drip 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 7fe0e76a3ce8cc23242eba6e08c6800eebbfbe18
-  data.tar.gz: fd19e70695e3da3b1ec9cccd4494d0209865285e
+SHA256:
+  metadata.gz: ccc378c36cdf4c16d9f69116b4921264a81c861ed30fd27453ed6c52f257b03c
+  data.tar.gz: 1f9f813e80c22eb8788289b79e59ed53e96e70fe32d6c572cc905274a3906441
 SHA512:
-  metadata.gz: 6b7fd18f94adc3263e4523fdd000ba77904201745daf725a9f6a9f8469c92b0d6382a23ebc7ebe0739599327578d559787691e948ad151223e8270db1ab0e817
-  data.tar.gz: 15a8fbf484bbcd9e19b29d945fba8c6ac4472711b417ad5394d7e787d5de04c83beb0ce269f16ecedfeffd46f7681101ffc94657035bc18160e14e4dbf18925e
+  metadata.gz: '038ac25011da60f555b56a65b69060befe154f938263a0cb3cbd541324cee6841d2a7cf482b1b1b3027152fd395fa9947d5a995a0b506a1ecd279de1dc75f5e8'
+  data.tar.gz: 0b93f499b391e0a5d376f851746ba2c44dac5d5a2dba18016050cddf769a0c9ff75f9efc818c24267c1131751d1dd583e80c9eec69afdcb4dc3a16f1c393993b

data/drip.gemspec

@@ -12,7 +12,9 @@ Gem::Specification.new do |s|
   s.description = ""
 
   s.rubyforge_project = "drip"
-  s.add_dependency "rbtree"  
+  s.add_dependency "rbtree"
+  s.add_development_dependency "test-unit"
+  s.add_development_dependency "sqlite3"
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }

data/lib/drip/version.rb

@@ -1,3 +1,3 @@
 class Drip
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/drip3.rb

@@ -0,0 +1,410 @@
+require 'drb/drb'
+require 'rinda/tuplespace'
+require 'enumerator'
+require 'sqlite3'
+
+class SQLite3::Database
+  def statement(s)
+    if @cache.nil?
+      @cache = Hash.new { |h, k| h[k] = self.prepare(k) }
+    end
+    @cache[s]
+  end
+end
+
+class SQLite3::Statement
+  def get_first_row(*bind_vars)
+    execute(*bind_vars).first
+  end
+
+  def get_first_value(*bind_vars)
+    execute(*bind_vars).each { |row| return row[0] }
+    nil
+  end
+end
+
+class Drip3
+  include DRbUndumped
+  INF = 1.0/0.0
+
+  def inspect; to_s; end
+  
+  def initialize(dir)
+    @event = Rinda::TupleSpace.new(5)
+    @event.write([:last, 1])
+    setup_db(dir)
+  end
+
+  def self.time_to_key(time)
+    time.tv_sec * 1000000 + time.tv_usec
+  end
+
+  def time_to_key(time)
+    self.class.time_to_key(time)
+  end
+
+  def self.key_to_time(key)
+    Time.at(*key.divmod(1000000))
+  end
+
+  def key_to_time(key)
+    self.class.key_to_time(key)
+  end
+
+  def write(obj, tag=nil)
+    write_after(Time.now, obj, tag)
+  end
+
+  def write_after(at, obj, tag=nil)
+    make_key(at) do |key|
+      do_write(key, obj, tag)
+    end
+  end
+
+  def write_at(at, obj, tag=nil)
+    make_key_at(at) do |key|
+      do_write(key, obj, tag)
+    end
+  end
+
+  def write_if_latest(cond, obj, tag=nil)
+    make_key(Time.now) do |key|
+      do_write(key, obj, tag, cond)
+    end
+  end
+
+  def fetch(key)
+    do_fetch(key)
+  end
+  alias [] fetch
+
+  def head(n=1, tag=nil)
+    if tag
+      do_head_tag(n, tag)
+    else
+      do_head(n)
+    end
+  end
+
+  def read(key, n=1, at_least=1, timeout=nil)
+    key = time_to_key(Time.now) unless key
+    ary = do_read(key, n)
+    while ary.size < at_least
+      key = ary[-1][0] unless ary.empty?
+      begin
+        renewer = make_renewer(timeout)
+        wait(key, renewer)
+      rescue Rinda::RequestExpiredError
+        return ary
+      end
+      ary += do_read(key, n - ary.size)
+    end
+    ary
+  end
+
+  def read_tag(key, tag, n=1, at_least=1, timeout=nil)
+    at_least = n if n < at_least
+    key = time_to_key(Time.now) unless key
+    ary = do_read_tag(key, tag, n)
+    while ary.size < at_least
+      key = ary[-1][0] unless ary.empty?
+      begin
+        renewer = make_renewer(timeout)
+        wait_tag(key, tag, renewer)
+      rescue Rinda::RequestExpiredError
+        return ary
+      end
+      ary += do_read_tag(key, tag, n - ary.size)
+    end
+    ary
+  end
+
+  def latest?(key, tag=nil)
+    do_latest(key, tag)
+  end
+
+  def older(key, tag=nil)
+    do_older(key, tag)
+  end
+
+  def newer(key, tag=nil)
+    return read(key, 1, 0)[0] unless tag
+    read_tag(key, tag, 1, 0)[0]
+  end
+
+  def tag_next(tag)
+    do_tag_next(tag)
+  end
+
+  def tag_prev(tag)
+    do_tag_prev(tag)
+  end
+
+  private
+  def transaction_rw(key, &blk)
+    _, db = @event.take([key, nil])
+    db.transaction do
+      return yield(db)
+    end
+  ensure
+    @event.write([key, db])
+  end
+
+  def transaction(&blk)
+    transaction_rw(:db, &blk)
+  end
+
+  def transaction_w(&blk)
+    transaction_rw(@w_key, &blk)
+  end
+
+  def from_row(ary)
+    return nil unless ary
+    return ary[0], Marshal.load(ary[1]), ary[2]
+  end
+
+  def do_fetch(key)
+    transaction do |db|
+      db.statement('select value from Drip where key=?').execute(key).each do |row|
+        return Marshal.load(row[0])
+      end
+    end
+    nil
+  end
+
+  def do_tag_next(tag)
+    return nil if tag.nil?
+    transaction do |db|
+      sql = <<SQL
+select tag from Drip where tag>? order by key asc limit 1
+SQL
+      db.statement(sql).get_first_value(tag) rescue nil
+    end
+  end
+  
+  def do_tag_prev(tag)
+    return nil if tag.nil?
+    transaction do |db|
+      sql = <<SQL
+select tag from Drip where tag<? order by key desc limit 1
+SQL
+      db.statement(sql).get_first_value(tag) rescue nil
+    end
+  end
+  
+  def do_head_tag(n, tag)
+    transaction do |db|
+      sql = <<SQL
+select key, value, tag from Drip where tag=? order by key desc limit ?;
+SQL
+      db.statement(sql).execute(tag, n).collect {|row| from_row(row)}.reverse
+    end
+  end
+
+  def do_read_tag(key, tag, n=1)
+    transaction do |db|
+      sql = <<SQL
+select key, value, tag from Drip
+  where key > ? and tag=?
+    order by key asc limit ?;
+SQL
+      db.statement(sql).execute(key, tag, n).collect {|row| from_row(row)}
+    end
+  end
+
+  def wait(key, renewer)
+    @event.read([:last, key+1 .. INF], renewer)[1]
+  end
+
+  def wait_tag(key, tag, renewer)
+    @event.read([tag, key+1 .. INF], renewer)[1]
+  end
+
+  def make_renewer(timeout)
+    case timeout
+    when 0
+      return 0
+    when Numeric
+      return Renewer.new(timeout)
+    else
+      nil
+    end
+  end
+
+  def do_head(n)
+    transaction do |db|
+      sql = <<SQL
+select key, value, tag from Drip order by key desc limit ?
+SQL
+      db.statement(sql).execute(n).collect {|row| from_row(row)}.reverse
+    end
+  end
+  
+  def do_read(key, n)
+    transaction do |db|
+      sql = <<SQL
+select key, value, tag from Drip where key > ? order by key asc limit ?
+SQL
+      db.statement(sql).execute(key, n).collect {|row| from_row(row)}
+    end
+  end
+
+  def do_older(key, tag)
+    if key.nil?
+      if tag.nil?
+        return do_head(1)[0]
+      else
+        return do_head_tag(1, tag)[0]
+      end
+    end
+    
+    transaction do |db|
+      if tag
+        sql = <<SQL
+select key, value, tag from Drip where key < ? and tag=?
+  order by key desc limit 1
+SQL
+        from_row(db.statement(sql).get_first_row(key, tag))
+      else
+        sql = <<SQL
+select key, value, tag from Drip where key < ?
+  order by key desc limit 1
+SQL
+        from_row(db.statement(sql).get_first_row(key))
+      end
+    end
+  end
+
+  def make_key(at=Time.now)
+    synchronize do |last|
+      key = [time_to_key(at), last + 1].max
+      yield(key)
+      # key
+    end
+  end
+  
+  def make_key_at(at)
+    synchronize do |last|
+      key = time_to_key(at)
+      raise 'InvalidTimeError' if key <= last
+      yield(key)
+      # key
+    end
+  end
+  
+  def synchronize
+    _, last = @event.take([:last, nil])
+    last = yield(last)
+  ensure
+    @event.write([:last, last])
+  end
+
+  def do_latest(key, tag)
+    transaction do |db|
+      do_latest_inner(db, key, tag)
+    end
+  end
+  
+  def do_latest_inner(db, key, tag)
+    if tag
+      sql = <<SQL
+select max(key) from Drip where tag=?
+SQL
+      v = db.statement(sql).get_first_value(tag) || 0
+    else
+      sql = <<SQL
+select max(key) from Drip
+SQL
+      v = db.statement(sql).get_first_value() || 0
+    end
+    return v == key
+  end
+  
+  def do_write(key, obj, tag, cond=nil)
+    unless tag.nil?
+      raise(ArgumentError) unless String === tag
+      tag = tag.to_s
+    end
+    transaction_w do |db|
+      if cond
+        cond.each {|it|
+          return nil unless do_latest_inner(db, it[1], it[0])
+        }
+      end
+      
+      if tag
+        @event.take([tag, nil], 0) rescue nil
+        @event.write([tag, key])
+      end
+      sql = 'insert into Drip values (?, ?, ?)'
+      db.statement(sql).execute(key, Marshal.dump(obj), tag)
+    end
+    key
+  end
+
+  private
+  def create_db(dir)
+    if dir
+      Dir.mkdir(dir) rescue nil
+      # fname = 'file:' + File.join(dir, 'drip.db') + '?cache=shared'
+      fname = File.join(dir, 'drip.db')
+      @w_key = :db
+      @event.write([:db, SQLite3::Database.open(fname)])
+    else
+      fname = ':memory:'
+      @w_key = :db
+      @event.write([:db, SQLite3::Database.open(fname)])
+    end
+  end
+
+  def setup_db(dir)
+    create_db(dir)
+    create_table
+  end
+  
+  def create_table
+    transaction_w do |db|
+      begin
+        db.execute('create table Drip (key bigint, value blob, tag text);')
+        db.execute('create index DripKey on Drip(key);')
+        db.execute('create index Tags on Drip(key, tag);')
+      rescue SQLite3::SQLException
+        unless $!.message == 'table Drip already exists'
+          raise $!
+        end
+      end
+    end
+  end
+
+  class Renewer
+    def initialize(timeout)
+      @at = Time.now + timeout
+    end
+    
+    def renew
+      @at - Time.now
+    end
+  end
+end
+
+if __FILE__ == $0
+  # d3 = Drip3.new('test_db')
+  d3 = Drip3.new(nil)
+
+  d3.write("こんにちはせかい\0\0こんにちはアゲイン")
+  p d3.head[0][1]
+
+  key = d3.write('start')
+  50.times do |n|
+    Thread.new(n) do |x|
+      100.times do
+        d3.write(x, 'count')
+      end
+      d3.write(x, 'wakeup')
+    end
+  end
+  50.times do
+    key, value, = d3.read_tag(key, 'wakeup', 1)[0]
+    p value
+  end
+end

data/test/basic3.rb

@@ -0,0 +1,250 @@
+require 'test/unit'
+require '../lib/drip3.rb'
+require 'fileutils'
+
+class TestDrip < Test::Unit::TestCase
+  def setup
+    @drip = Drip3.new(nil)
+  end
+
+  def test_time_travel
+    @drip.write('age' => 1)
+    @drip.write('age' => 2)
+    @drip.write('age' => 3)
+    oid, value = @drip.older(nil)
+    assert_equal(value, 'age' => 3)
+    oid, value = @drip.older(oid)
+    assert_equal(value, 'age' => 2)
+    oid, value = @drip.older(oid)
+    assert_equal(value, 'age' => 1)
+    oid, value = @drip.older(oid)
+    assert_equal(oid, nil)
+    assert_equal(value, nil)
+    
+    oid, value = @drip.newer(0)
+    assert_equal(value, 'age' => 1)
+    oid, value = @drip.newer(oid)
+    assert_equal(value, 'age' => 2)
+    oid, value = @drip.newer(oid)
+    assert_equal(value, 'age' => 3)
+    oid, value = @drip.newer(oid)
+    assert_equal(oid, nil)
+    assert_equal(value, nil)
+  end
+
+  def test_read
+    11.times do |n|
+      @drip.write("n=#{n}" => 'x' * n, n => n, "n" => n, :symbol => n)
+    end
+    ary = @drip.read(0, 3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[1][1]['n'], 1)
+    assert_equal(ary[2][1]['n'], 2)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 2)
+
+    oid = @drip.write('latest', 'tag1')
+    oid, value, *tags = @drip.newer(oid - 1)
+    assert_equal(value, 'latest')
+    assert_equal(tags, ['tag1'])
+  end
+
+  def test_symbol_is_not_tag
+    assert_raise(ArgumentError){
+      @drip.write({:symbol => :symbol, 'string' => :string}, :symbol)
+    }
+  end
+
+  def test_number_is_not_tag
+    assert_raise(ArgumentError){
+      @drip.write({5 => :five, 'string' => :string}, 5)
+    }
+  end
+  
+  def test_older_now_is_newest
+    @drip.write('age' => 1)
+    @drip.write('age' => 2)
+    @drip.write('age' => 3)
+    oid, value, = @drip.older(nil)
+    assert_equal(value, 'age' => 3)
+    oid, value, = @drip.older(@drip.time_to_key(Time.now))
+    assert_equal(value, 'age' => 3)
+
+    # newer(past)
+    assert_equal(@drip.newer(0)[1], 'age' => 1)
+    assert_equal(@drip.newer(0)[1], 'age' => 1)
+  end
+
+  def test_read_tag
+    3.times do |n|
+      @drip.write({'n' => n}, 'n')
+      @drip.write({'n' => n, '2' => n * 2}, 'n')
+      @drip.write({'n' => n, '2' => n * 2, '3' => n * 3}, 'n')
+    end
+    
+    ary = @drip.read_tag(0, 'n', 10)
+    assert_equal(ary.size, 9)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[0][2], 'n')
+  end
+
+  def test_head
+    10.times do |n|
+      @drip.write(n)
+    end
+    
+    ary = @drip.head(3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1], 7)
+    assert_equal(ary[2][1], 9)
+  end
+  
+  def test_after
+    assert_equal(@drip.write_after(Time.at(1), 1), 1000000)
+    assert_equal(@drip.write_after(Time.at(2), 2), 2000000)
+    assert_equal(@drip.write_after(Time.at(3), 3), 3000000)
+    assert_equal(@drip.write_after(Time.at(4), 4), 4000000)
+    assert_equal(@drip.write_after(Time.at(4), 5), 4000001)
+    assert_equal(@drip.write_after(Time.at(2), 6), 4000002)
+    assert_equal(@drip.write_after(Time.at(5), 6), 5000000)
+    assert_equal(@drip.write_after(Time.at(5), 7), 5000001)
+    assert_equal(@drip.write_at(Time.at(6), 8), 6000000)
+    assert_raise(RuntimeError) {@drip.write_at(Time.at(6), 8)}
+    assert_equal(@drip.write_after(Time.at(5), 8), 6000001)
+  end
+  
+  def test_use_string_tag
+    oid = @drip.write(:symbol, 'dup')
+    assert_equal(@drip[oid], :symbol)
+  end
+
+  def test_latest?
+    key = @drip.write(:start)
+
+    assert_equal(@drip.latest?(key), true)
+
+    10.times do |n|
+      @drip.write(n)
+    end
+    assert_equal(@drip.latest?(key), false)
+    key = @drip.write(:stop)
+    assert_equal(@drip.latest?(key), true)
+
+    key = @drip.write(:tag_start, 'tag')
+    @drip.write(:tag, 'ignore tag')
+    assert_equal(@drip.latest?(key, 'tag'), true)
+    @drip.write(:tag, 'tag')
+    assert_equal(@drip.latest?(key, 'tag'), false)
+  end
+
+  def test_write_if_latest
+    t1 = @drip.write('t1', 't1')
+    t2 = @drip.write('t2', 't2')
+    t3 = @drip.write('t3', 't3')
+    assert_equal(@drip.latest?(t1, 't1'), true)
+    assert(@drip.write_if_latest([['t1', t1],
+                                  ['t2', t2],
+                                  ['t3', t3],
+                                  ['t4', 0]], 'hello', 't1'))
+    assert_equal(@drip.latest?(t1, 't1'), false)
+    assert_equal(@drip.write_if_latest([['t1', t1],
+                                        ['t2', t2],
+                                        ['t3', t3]], 'hello', 't1'), nil)
+  end
+
+end
+
+class TestDripUsingStorage < TestDrip
+  def remove_drip(dir='test_db')
+    FileUtils.rm_r(dir, :force => true)
+  end
+
+  def setup
+    remove_drip
+    @drip = Drip3.new('test_db')
+  end
+
+  def teardown
+    remove_drip
+  end
+
+  def test_tag_browse
+    @drip.write(1, 't1')
+    @drip.write(2, 't2')
+    @drip.write(3, 't3')
+    @drip.write(4, 't4')
+
+    assert_equal(@drip.tag_next(''), 't1')
+    assert_equal(@drip.tag_next('t1'), 't2')
+    assert_equal(@drip.tag_next('t3'), 't4')
+    assert_equal(@drip.tag_next('t5'), nil)
+
+    assert_equal(@drip.tag_prev('u'), 't4')
+    assert_equal(@drip.tag_prev('t2'), 't1')
+    assert_equal(@drip.tag_prev('t1'), nil)
+  end
+
+  def test_twice_latest?
+    assert_equal(@drip.latest?(1), false)
+    tag1 = @drip.write('tag1', 'tag1')
+    assert_equal(@drip.latest?(tag1), true)
+    @drip.write('nop', 'tag1')
+    @drip.write('nop', 'tag1')
+    tag2 = @drip.write('tag2', 'tag1')
+    assert_equal(@drip.latest?(1), false)
+    drip = Drip3.new('test_db')
+    assert_equal(drip.latest?(1), false)
+    assert_equal(drip.latest?(tag1, 'tag1'), false)
+    assert_equal(drip.latest?(tag2, 'tag1'), true)
+    assert_equal(drip.latest?(tag2, 'tag0'), false)
+  end
+  
+  def test_twice
+    11.times do |n|
+      @drip.write("n=#{n}" => 'x' * n, n => n, "n" => n, :symbol => n)
+    end
+    
+    drip = Drip3.new('test_db')
+    ary = drip.head(3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1]['n'], 8)
+    assert_equal(ary[1][1]['n'], 9)
+    assert_equal(ary[2][1]['n'], 10)
+    ary = drip.head(1)
+    assert_equal(ary.size, 1)
+    assert_equal(ary[0][1]['n'], 10)
+    ary = drip.read(0, 3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[1][1]['n'], 1)
+    assert_equal(ary[2][1]['n'], 2)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 2)
+  end
+
+  def test_huge
+    str = File.read(__FILE__)
+
+    10.times do 
+      Thread.new do
+        1000.times do |n|
+          @drip.write(str, "n=#{n}")
+        end
+      end
+    end
+
+    assert_equal(@drip.read_tag(0, 'n=999', 10, 10).size, 10)
+
+    assert_equal(10000, @drip.read(0, 12000, 10000).size)
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: drip
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Masatoshi Seki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-17 00:00:00.000000000 Z
+date: 2018-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbtree
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email: 
 executables: []
@@ -39,6 +67,7 @@ files:
 - install.rb
 - lib/drip.rb
 - lib/drip/version.rb
+- lib/drip3.rb
 - lib/my_drip.rb
 - sample/copocopo.rb
 - sample/demo4book/crawl.rb
@@ -54,6 +83,7 @@ files:
 - sample/simple-oauth.rb
 - sample/tw_markov.rb
 - test/basic.rb
+- test/basic3.rb
 homepage: https://github.com/seki/Drip
 licenses: []
 metadata: {}
@@ -73,9 +103,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: drip
-rubygems_version: 2.5.1
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Simple RD-Stream for Rinda::TupleSpace lovers.
 test_files:
 - test/basic.rb
+- test/basic3.rb