drip 0.0.1

data/sample/hello_tw.rb

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+require 'drip_tw'
+
+app = DripDemo.new
+app.update("@m_seki なんで？", 76285215248617472)

data/sample/my_status.rb

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+require 'drip_tw'
+
+def last_tweet_id
+  key = nil
+  while kv = MyDrip.older(key, 'DripDemo MyStatus')
+    key, value = kv
+    return value['id'] if value['id']
+  end
+  nil
+end
+
+def get_user_timeline(app, since_id, max_id)
+  count = 3
+  begin
+    ary = app.user_timeline(since_id, max_id)
+  rescue
+    p $!
+    count -= 1
+    return nil, [] if count <= 0
+    sleep 5
+    retry
+  end
+  max_id = nil
+  ary.reverse_each do |event|
+    next unless event['id']
+    max_id = event['id'] - 1
+    break
+  end
+  return max_id, ary
+end
+
+app = DripDemo.new
+
+since_id = last_tweet_id
+
+max_id = nil
+timeline = []
+while true
+  max_id, ary = get_user_timeline(app, since_id, max_id)
+  break if ary.empty?
+  pp [ary[0]['text'], ary[-1]['text']] rescue nil
+  sleep 5
+  timeline += ary
+end
+
+p [timeline.size, 'continue?']
+gets
+
+timeline.reverse_each do |event|
+  MyDrip.write(event, 'DripDemo MyStatus')
+end

data/sample/simple-oauth.rb

@@ -0,0 +1,140 @@
+#
+#     import from https://github.com/shibason/rb-simple-oauth
+#     see: http://d.hatena.ne.jp/shibason/20090809/1249764381
+require 'uri'
+require 'net/http'
+require 'openssl'
+
+class SimpleOAuth
+  def initialize(consumer_key, consumer_secret, token, token_secret)
+    @consumer_key = consumer_key
+    @consumer_secret = consumer_secret
+    @token = token
+    @token_secret = token_secret
+    # This class supports only 'HMAC-SHA1' as signature method at present.
+    @signature_method = 'HMAC-SHA1'
+  end
+
+  def get(url, headers = {})
+    request(:GET, url, nil, headers)
+  end
+
+  def head(url, headers = {})
+    request(:HEAD, url, nil, headers)
+  end
+
+  def post(url, body = nil, headers = {})
+    request(:POST, url, body, headers)
+  end
+
+  def put(url, body = nil, headers = {})
+    request(:PUT, url, body, headers)
+  end
+
+  def delete(url, headers = {})
+    request(:DELETE, url, nil, headers)
+  end
+
+private
+  def request(method, url, body = nil, headers = {})
+    method = method.to_s
+    url = URI.parse(url)
+    request = create_http_request(method, url.request_uri, body, headers)
+    request['Authorization'] = auth_header(method, url, request.body)
+    Net::HTTP.new(url.host, url.port).request(request)
+  end
+
+  RESERVED_CHARACTERS = /[^a-zA-Z0-9\-\.\_\~]/
+
+  def escape(value)
+    URI.escape(value.to_s, RESERVED_CHARACTERS)
+  end
+
+  def encode_parameters(params, delimiter = '&', quote = nil)
+    if params.is_a?(Hash)
+      params = params.map do |key, value|
+        "#{escape(key)}=#{quote}#{escape(value)}#{quote}"
+      end
+    else
+      params = params.map { |value| escape(value) }
+    end
+    delimiter ? params.join(delimiter) : params
+  end
+
+  VERSION = '0.1'
+  USER_AGENT = "SimpleOAuth/#{VERSION}"
+
+  def create_http_request(method, path, body, headers)
+    method = method.capitalize.to_sym
+    request = Net::HTTP.const_get(method).new(path, headers)
+    request['User-Agent'] = USER_AGENT
+    if method == :Post || method == :Put
+      request.body = body.is_a?(Hash) ? encode_parameters(body) : body.to_s
+      request.content_type = 'application/x-www-form-urlencoded'
+      request.content_length = (request.body || '').length
+    end
+    request
+  end
+
+  def auth_header(method, url, body)
+    parameters = oauth_parameters
+    parameters[:oauth_signature] = signature(method, url, body, parameters)
+    'OAuth ' + encode_parameters(parameters, ', ', '"')
+  end
+
+  OAUTH_VERSION = '1.0'
+
+  def oauth_parameters
+    {
+      :oauth_consumer_key => @consumer_key,
+      :oauth_token => @token,
+      :oauth_signature_method => @signature_method,
+      :oauth_timestamp => timestamp,
+      :oauth_nonce => nonce,
+      :oauth_version => OAUTH_VERSION
+    }
+  end
+
+  def timestamp
+    Time.now.to_i.to_s
+  end
+
+  def nonce
+    OpenSSL::Digest::Digest.hexdigest('MD5', "#{Time.now.to_f}#{rand}")
+  end
+
+  def signature(*args)
+    base64(digest_hmac_sha1(signature_base_string(*args)))
+  end
+
+  def base64(value)
+    [ value ].pack('m').gsub(/\n/, '')
+  end
+
+  def digest_hmac_sha1(value)
+    OpenSSL::HMAC.digest(OpenSSL::Digest::SHA1.new, secret, value)
+  end
+
+  def secret
+    escape(@consumer_secret) + '&' + escape(@token_secret)
+  end
+
+  def signature_base_string(method, url, body, parameters)
+    method = method.upcase
+    base_url = signature_base_url(url)
+    parameters = normalize_parameters(parameters, body, url.query)
+    encode_parameters([ method, base_url, parameters ])
+  end
+
+  def signature_base_url(url)
+    URI::HTTP.new(url.scheme, url.userinfo, url.host, nil, nil, url.path,
+                  nil, nil, nil)
+  end
+
+  def normalize_parameters(parameters, body, query)
+    parameters = encode_parameters(parameters, nil)
+    parameters += body.split('&') if body
+    parameters += query.split('&') if query
+    parameters.sort.join('&')
+  end
+end

data/sample/tw_markov.rb

@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+require 'my_drip'
+require 'MeCab'
+require 'pp'
+
+class Bag
+  def initialize
+    @bag = Hash.new(0)
+    @size = 0
+    @cache = nil
+  end
+  attr_reader :size
+  
+  def push(obj)
+    @cache = nil
+    @bag[obj] += 1
+    @size += 1
+    obj
+  end
+
+  def sample
+    @cache = @bag.sort_by {|k, v| v} unless @cache
+    prob = rand(@size)
+    @cache.each do |k, v|
+      prob -= v
+      return k if prob < 0
+    end
+  end
+end
+
+class BagInBag < Bag
+  def initialize(&blk)
+    super
+    @bag_in_bag = Hash.new {|h, k| h[k] = blk.call}
+  end
+  
+  def push(obj, *rest)
+    super(obj)
+    @bag_in_bag[obj].push(*rest)
+  end
+
+  def [](key, *rest)
+    if rest.empty?
+      @bag_in_bag[key]
+    else
+      @bag_in_bag[key][*rest]
+    end
+  end
+end
+
+class EntityEnum
+  def initialize(str, entity)
+    @str = str || ''
+    @entity = []
+    (entity || []).each do |k, v|
+      @entity += v
+    end
+  end
+
+  def each
+    unless @entity
+      yield(@str, nil)
+      return
+    end
+    ary = @entity.sort_by {|e| e['indices'][0]}
+    cursor = 0
+    while head = ary.shift
+      left, right = head['indices']
+      if left > cursor
+        yield(@str[cursor ... left], nil)
+      end
+      yield(@str[left .. right], head)
+      cursor = right + 1
+    end
+    if @str.length > cursor
+      yield(@str[cursor ... @str.length], nil)
+    end
+  end
+end
+
+class Markov
+  def initialize
+    @bag = BagInBag.new { BagInBag.new { Bag.new } }
+    @start = Bag.new
+    @intern = Hash.new
+  end
+
+  def intern(str)
+    @intern[str] ||= str
+    @intern[str]
+  end
+
+  def add(ary)
+    return if ary.size <= 3
+    ary = ary.collect {|s| intern(s)}
+    @start.push(ary[0])
+    ary.each_cons(3) do |p0, p1, suffix|
+      @bag.push(p0, p1, suffix)
+    end
+  end
+
+  def generate(max_len=30)
+    s = @start.sample
+    ary = [s, @bag[s].sample]
+    max_len.times do
+      it = @bag[*ary[-2, 2]].sample
+      break if it == :eos
+      ary << it
+    end
+    ary
+  end
+end
+
+$screen_name = {}
+
+def mention_user
+  " @#{$screen_name.keys.sample} "
+end
+
+def as_str(ary)
+  ary = ary.map {|x| x == :name ? mention_user : x}
+  ary.inject{|s,c|s+=(s+"\00"+c=~/\w\00\w/?" ":"")+c}
+end
+
+markov = Markov.new
+m = MeCab::Tagger.new('-Owakati')
+
+MyDrip.head(4000, 'DripDemo MyStatus').each do |k, v|
+  next if v['retweeted_status']
+  entity = v['entities']
+  str = v['text']
+  ary = []
+  EntityEnum.new(str, entity).each do |s, kind|
+    next if s == 'RT'
+    unless kind
+      ary += m.parse(s).split(' ')
+    else
+      ary <<:name if kind['screen_name']
+      if kind['screen_name']
+        $screen_name[kind['screen_name']] = true
+      end
+    end
+  end
+  markov.add(ary + [:eos])
+end
+
+while gets
+  10.times do
+    ary = markov.generate
+    next if ary.size < 10
+    puts as_str(ary)
+    puts
+  end
+end

data/test/basic.rb

@@ -0,0 +1,156 @@
+require 'test/unit'
+require '../lib/drip.rb'
+require 'fileutils'
+
+class TestDrip < Test::Unit::TestCase
+  def setup
+    @drip = Drip.new(nil)
+  end
+
+  def test_time_travel
+    @drip.write('age' => 1)
+    @drip.write('age' => 2)
+    @drip.write('age' => 3)
+    oid, value = @drip.older(nil)
+    assert_equal(value, 'age' => 3)
+    oid, value = @drip.older(oid)
+    assert_equal(value, 'age' => 2)
+    oid, value = @drip.older(oid)
+    assert_equal(value, 'age' => 1)
+    oid, value = @drip.older(oid)
+    assert_equal(oid, nil)
+    assert_equal(value, nil)
+    
+    oid, value = @drip.newer(0)
+    assert_equal(value, 'age' => 1)
+    oid, value = @drip.newer(oid)
+    assert_equal(value, 'age' => 2)
+    oid, value = @drip.newer(oid)
+    assert_equal(value, 'age' => 3)
+    oid, value = @drip.newer(oid)
+    assert_equal(oid, nil)
+    assert_equal(value, nil)
+  end
+
+  def test_read
+    11.times do |n|
+      @drip.write("n=#{n}" => 'x' * n, n => n, "n" => n, :symbol => n)
+    end
+    ary = @drip.read(0, 3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[1][1]['n'], 1)
+    assert_equal(ary[2][1]['n'], 2)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = @drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 2)
+
+    oid = @drip.write('latest', 'tag1', 'tag2')
+    oid, value, *tags = @drip.newer(oid - 1)
+    assert_equal(value, 'latest')
+    assert_equal(tags, ['tag1', 'tag2'])
+  end
+
+  def test_symbol_is_not_tag
+    @drip.write({:symbol => :symbol, 'string' => :string}, :symbol, 'string')
+    assert_raise(ArgumentError) {@drip.read_tag(0, :symbol, 1, 0)}
+    oid, value = @drip.older(@drip.time_to_key(Time.now))
+    assert_equal(value, {:symbol => :symbol, 'string' => :string})
+  end
+
+  def test_number_is_not_tag
+    @drip.write({5 => :five, 'string' => :string}, 5, 'string')
+    assert_equal(@drip.read_tag(0, 'string', 1, 0).size, 1)
+    assert_raise(ArgumentError) {@drip.read_tag(0, 5, 1, 0)}
+  end
+  
+  def test_older_now_is_newest
+    @drip.write('age' => 1)
+    @drip.write('age' => 2)
+    @drip.write('age' => 3)
+    oid, value = @drip.older(nil)
+    assert_equal(value, 'age' => 3)
+    oid, value = @drip.older(@drip.time_to_key(Time.now))
+    assert_equal(value, 'age' => 3)
+
+    # newer(past)
+    assert_equal(@drip.newer(0)[1], 'age' => 1)
+    assert_equal(@drip.newer(0)[1], 'age' => 1)
+  end
+
+  def test_read_tag
+    3.times do |n|
+      @drip.write({'n' => n}, 'n')
+      @drip.write({'n' => n, '2' => n * 2}, 'n', '2')
+      @drip.write({'n' => n, '2' => n * 2, '3' => n * 3}, 'n', '2', '3')
+    end
+    
+    ary = @drip.read_tag(0, 'n', 10)
+    assert_equal(ary.size, 9)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[0][2], 'n')
+    assert_equal(ary[2][4], '3')
+  end
+
+  def test_head
+    10.times do |n|
+      @drip.write(n)
+    end
+    
+    ary = @drip.head(3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1], 7)
+    assert_equal(ary[2][1], 9)
+  end
+  
+  def test_after
+    assert_equal(@drip.write_after(Time.at(1), 1), 1000000)
+    assert_equal(@drip.write_after(Time.at(2), 2), 2000000)
+    assert_equal(@drip.write_after(Time.at(3), 3), 3000000)
+    assert_equal(@drip.write_after(Time.at(4), 4), 4000000)
+    assert_equal(@drip.write_after(Time.at(4), 5), 4000001)
+    assert_equal(@drip.write_after(Time.at(2), 6), 4000002)
+    assert_equal(@drip.write_after(Time.at(5), 6), 5000000)
+    assert_equal(@drip.write_after(Time.at(5), 7), 5000001)
+    assert_equal(@drip.write_at(Time.at(6), 8), 6000000)
+    assert_raise(RuntimeError) {@drip.write_at(Time.at(6), 8)}
+    assert_equal(@drip.write_after(Time.at(5), 8), 6000001)
+  end
+end
+
+class TestDripUsingStorage < TestDrip
+  def remove_drip(dir='test_db')
+    FileUtils.rm_r(dir, :force => true)
+  end
+
+  def setup
+    remove_drip
+    @drip = Drip.new('test_db')
+  end
+
+  def teardown
+    remove_drip
+  end
+  
+  def test_twice
+    11.times do |n|
+      @drip.write("n=#{n}" => 'x' * n, n => n, "n" => n, :symbol => n)
+    end
+
+    drip = Drip.new('test_db')
+    ary = drip.read(0, 3)
+    assert_equal(ary.size, 3)
+    assert_equal(ary[0][1]['n'], 0)
+    assert_equal(ary[1][1]['n'], 1)
+    assert_equal(ary[2][1]['n'], 2)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 3)
+    ary = drip.read(ary[2][0], 3)
+    assert_equal(ary.size, 2)
+  end
+end