drip 0.0.1 → 0.0.2

data/lib/drip.rb

@@ -7,6 +7,127 @@ class Drip
   include DRbUndumped
   def inspect; to_s; end
 
+  class ImmutableDrip
+    class Generator
+      def initialize
+        @pool = []
+        @tag = []
+        @shared = Hash.new {|h, k| h[k] = k; k}
+      end
+
+      def add(key, value, *tag)
+        @pool << [key, value]
+        idx = @pool.size - 1
+        tag.uniq.each do |t|
+          @tag << [[@shared[t], key], idx]
+        end
+      end
+      
+      def generate
+        tag = @tag.sort
+        tag.inject(nil) do |last, kv|
+          k = kv[0]
+          k[0] = last if k[0] == last
+          k[0]
+        end
+        ImmutableDrip.new(@pool.sort, tag)
+      end
+    end
+
+    INF = 1.0/0
+
+    def initialize(pool=[], tag=[])
+      @pool = pool
+      @tag = tag
+    end
+
+    def fetch(key)
+      idx = lower_boundary(@pool, key)
+      k, v = @pool[idx]
+      k == key ? v.to_a : nil
+    end
+
+    def read(key, n=1)
+      idx = lower_boundary(@pool, key + 1)
+      return [] unless idx
+      @pool[idx, n].collect {|kv|
+        [kv[0], *kv[1].to_a]
+      }
+    end
+
+    def read_tag(key, tag, n=1)
+      idx = lower_boundary(@tag, [tag, key + 1])
+      return [] unless idx
+      @tag[idx, n].find_all {|kv| kv[0][0] == tag}.collect {|kv| 
+        [kv[0][1], *@pool[kv[1]][1].to_a]
+      }
+    end
+
+    def head_tag(n, tag)
+      lower = lower_boundary(@tag, [tag, 0])
+      upper = upper_boundary(@tag, [tag, INF])
+      lower = [lower, upper - n].max
+      @tag[lower ... upper].collect {|kv|
+        [kv[0][1], *@pool[kv[1]][1].to_a]
+      }
+    end
+
+    def head(n=1, tag=nil)
+      return head_tag(n, tag) if tag
+      n = @pool.size < n ? @pool.size : n
+      @pool[-n, n].collect {|kv|
+        [kv[0], *kv[1].to_a]
+      }
+    end
+
+    def older_tag(key, tag)
+      idx = upper_boundary(@tag, [tag, key-1])
+      k, v = @tag[idx - 1]
+      k && k[0] == tag ? [k[1], *@pool[v][1].to_a] : nil
+    end
+
+    def older(key, tag=nil)
+      key = @pool[-1][0] + 1 unless key
+      return older_tag(key, tag) if tag
+      idx = upper_boundary(@pool, key - 1)
+      k, v = @pool[idx - 1]
+      k && k < key ? [k, *v.to_a] : nil
+    end
+
+    def newer(key, tag=nil)
+      return read(key, 1)[0] unless tag
+      read_tag(key, tag, 1)[0]
+    end
+    
+    def lower_boundary(ary, key)
+      lower = -1
+      upper = ary.size
+      while lower + 1 != upper
+        mid = (lower + upper).div(2)
+        if (ary[mid][0] <=> key) < 0
+          lower = mid
+        else
+          upper = mid
+        end
+      end
+      return upper
+    end
+    
+    def upper_boundary(ary, key)
+      lower = -1
+      upper = ary.size
+      while lower + 1 != upper
+        mid = (lower + upper).div(2)
+        if (ary[mid][0] <=> key) <= 0
+          lower = mid
+        else
+          upper = mid
+        end
+      end
+      return lower + 1
+    end
+  end
+
   def initialize(dir, option={})
     @pool = RBTree.new
     @tag = RBTree.new
@@ -15,20 +136,20 @@ class Drip
     prepare_store(dir, option)
   end
 
-  def write(*value)
-    write_after(Time.now, *value)
+  def write(obj, *tags)
+    write_after(Time.now, obj, *tags)
   end
 
   def write_after(at, *value)
     make_key(at) do |key|
-      do_write(key, value)
+      value = do_write(key, value)
       @pool[key] = @store.write(key, value)
     end
   end
   
   def write_at(at, *value)
     make_key_at(at) do |key|
-      do_write(key, value)
+      value = do_write(key, value)
       @pool[key] = @store.write(key, value)
     end
   end
@@ -197,15 +318,30 @@ class Drip
     end
 
     Dir.mkdir(dir) rescue nil
+    dump = Dir.glob(File.join(dir, '*.dump')).max_by do |fn| 
+      File.basename(fn).to_i(36)
+    end
+    if dump
+      @pool, @tag, last = File.open(dump, 'rb') {|fp| Marshal.load(fp)}
+      @event.take([:last, nil])
+      @event.write([:last, last])
+      File.unlink(dump)
+    end
+    loaded = dump ? File.basename(dump).to_i(36) : 0
     Dir.glob(File.join(dir, '*.log')) do |fn|
+      next if loaded > File.basename(fn).to_i(36)
       begin
         store = SimpleStore.reader(fn)
         restore(store)
       rescue
       end
     end
-    name = time_to_key(Time.now).to_s(36) + '.log'
-    @store = SimpleStore.new(File.join(dir, name))
+    name = time_to_key(Time.now).to_s(36)
+    _, last = @event.read([:last, nil])
+    File.open(File.join(dir, name + '.dump'), 'wb') {|fp|
+      Marshal.dump([@pool, @tag, last], fp)
+    }
+    @store = SimpleStore.new(File.join(dir, name + '.log'))
   end
 
   def shared_text(str)
@@ -218,13 +354,14 @@ class Drip
   end
 
   def do_write(key, value)
-    (1...value.size).each do |n|
-      k = value[n]
+    obj, *tags = value
+    tags.uniq!
+    tags.each do |k|
       next unless String === k
       tag = shared_text(k)
       @tag[[tag, key]] = key
     end
-    @pool[key] = value
+    @pool[key] = [obj] + tags
   end
 
   def restore(store)

data/lib/drip/version.rb

@@ -1,3 +1,3 @@
 module Drip
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/my_drip.rb

@@ -37,6 +37,10 @@ def MyDrip.invoke
   end
 end
 
+def MyDrip.inspect
+  "<MyDrip: #{@uri}>"
+end
+
 class DripCursor
   def initialize(drip, bufsiz=10, at_least=10)
     @drip = drip

data/sample/copocopo.rb

@@ -54,7 +54,7 @@ class CopoCopo
       @last, event = @drip.read_tag(@last, 'DripDemo Event', 1)[0]
       next if retweet?(event)
       next if mention?(event)
-      next unless Time.now < created_at(event) + 60000
+      next unless Time.now < created_at(event) + 6000
       name = dig(event, 'user', 'screen_name')
       next unless @friends.include?(name)
       ary = extract(event['text'] || '')

data/sample/demo4book/crawl.rb

@@ -0,0 +1,56 @@
+require 'pp'
+require 'my_drip'
+require 'monitor'
+
+class Crawler
+  include MonitorMixin
+
+  def initialize
+    super()
+    @root = File.expand_path('~/develop/git-repo/')
+    @drip = MyDrip
+    k, = @drip.head(1, 'rbcrawl-begin')[0]
+    @fence = k || 0
+  end
+
+  def last_mtime(fname)
+    k, v, = @drip.head(1, 'rbcrawl-fname=' + fname)[0]
+    (v && k > @fence) ? v[1] : Time.at(1)
+  end
+
+  def do_crawl
+    synchronize do
+      ary = []
+      Dir.chdir(@root)
+      Dir.glob('**/*.rb').each do |fname|
+        mtime = File.mtime(fname)
+        next if last_mtime(fname) >= mtime
+        @drip.write([fname, mtime, File.read(fname)],
+                    'rbcrawl', 'rbcrawl-fname=' + fname)
+        ary << fname
+      end
+      @drip.write(ary, 'rbcrawl-footprint')
+      ary
+    end
+  end
+  
+  def quit
+    synchronize do
+      exit(0)
+    end
+  end
+end
+
+if __FILE__ == $0
+  crawler = Crawler.new
+  Thread.new do
+    while true
+      pp crawler.do_crawl
+      sleep 60
+    end
+  end
+  
+  gets
+  crawler.quit
+end
+

data/sample/demo4book/demo_ui.rb

@@ -0,0 +1,71 @@
+require 'index'
+require 'crawl'
+require 'webrick/cgi'
+require 'erb'
+
+class DemoListView
+  include ERB::Util
+  extend ERB::DefMethod
+  def_erb_method('to_html(word, list)', ERB.new(<<EOS))
+<html><head><title>Demo UI</title></head><body>
+<form method="post"><input type="text" name="w" value="<%=h word %>" /></form>
+<% if word %>
+<p>search: <%=h word %></p>
+<ul>
+<%   list.each do |fname| %>
+<li><%=h fname%></li>
+<%   end %>
+</ul>
+<% end %>
+</body></html>
+EOS
+end
+
+class DemoUICGI < WEBrick::CGI
+  def initialize(crawler, indexer, *args)
+    super(*args)
+    @crawler = crawler
+    @indexer = indexer
+    @list_view = DemoListView.new
+  end
+
+  def req_query(req, key)
+    value ,= req.query[key]
+    return nil unless value
+    value.force_encoding('utf-8')
+    value
+  end
+
+  def do_GET(req, res)
+    if req.path_info == '/quit'
+      Thread.new do
+        @crawler.quit
+      end
+    end
+    word = req_query(req, 'w') || ''
+    list = word.empty? ? [] : @indexer.dict.query(word)
+    res['content-type'] = 'text/html; charset=utf-8'
+    res.body = @list_view.to_html(word, list)
+  end
+  
+  alias do_POST do_GET
+end
+
+if __FILE__ == $0
+  crawler = Crawler.new
+  Thread.new do
+    while true
+      pp crawler.do_crawl
+      sleep 60
+    end
+  end
+
+  indexer = Indexer.new
+  Thread.new do
+    indexer.update_dict
+  end
+  
+  cgi = DemoUICGI.new(crawler, indexer)
+  DRb.start_service('druby://localhost:50830', cgi)
+  DRb.thread.join
+end

data/sample/demo4book/demo_ui_webrick.rb

@@ -0,0 +1,69 @@
+require 'index'
+require 'crawl'
+require 'webrick'
+require 'erb'
+
+class DemoListView
+  include ERB::Util
+  extend ERB::DefMethod
+  def_erb_method('to_html(word, list)', ERB.new(<<EOS))
+<html><head><title>Demo UI</title></head><body>
+<form method="post"><input type="text" name="w" value="<%=h word %>" /></form>
+<% if word %>
+<p>search: <%=h word %></p>
+<ul>
+<%   list.each do |fname| %>
+<li><%=h fname%></li>
+<%   end %>
+</ul>
+<% end %>
+</body></html>
+EOS
+end
+
+class DemoUIServlet < WEBrick::HTTPServlet::AbstractServlet
+  def initialize(server, crawler, indexer, list_view)
+    super(server)
+    @crawler = crawler
+    @indexer = indexer
+    @list_view = list_view
+  end
+
+  def req_query(req, key)
+    value ,= req.query[key]
+    return nil unless value
+    value.force_encoding('utf-8')
+    value
+  end
+
+  def do_GET(req, res)
+    word = req_query(req, 'w') || ''
+    list = word.empty? ? [] : @indexer.dict.query(word)
+    res['content-type'] = 'text/html; charset=utf-8'
+    res.body = @list_view.to_html(word, list)
+  end
+  
+  alias do_POST do_GET
+end
+
+if __FILE__ == $0
+  crawler = Crawler.new
+  Thread.new do
+    while true
+      pp crawler.do_crawl
+      sleep 60
+    end
+  end
+
+  indexer = Indexer.new
+  Thread.new do
+    indexer.update_dict
+  end
+
+  server = WEBrick::HTTPServer.new({:Port => 10080,
+                                    :BindAddress => '127.0.0.1'})
+  server.mount('/', DemoUIServlet, crawler, indexer, DemoListView.new)
+  trap('INT') { server.shutdown }
+  server.start
+  crawler.quit
+end

data/sample/demo4book/index.rb

@@ -0,0 +1,96 @@
+require 'nkf'
+require 'rbtree'
+require 'my_drip'
+require 'monitor'
+require 'pp'
+
+
+class Indexer
+  def initialize(cursor=0)
+    @drip = MyDrip
+    @dict = Dict.new
+    k, = @drip.head(1, 'rbcrawl-begin')[0]
+    @fence = k || 0
+    @cursor = [cursor, @fence].max
+  end
+  attr_reader :dict
+
+  def update_dict
+    each_document do |cur, prev|
+      @dict.delete(*prev) if prev
+      @dict.push(*cur)
+    end
+  end
+
+  def each_document
+    while true
+      ary = @drip.read_tag(@cursor, 'rbcrawl', 10, 1)
+      ary.each do |k, v|
+        prev = prev_version(k, v[0])
+        yield(v, prev)
+        @cursor = k
+      end
+    end
+  end
+
+  def prev_version(cursor, fname)
+    k, v = @drip.older(cursor, 'rbcrawl-fname=' + fname)
+    (v && k > @fence) ? v : nil
+  end
+end
+
+class Dict
+  include MonitorMixin
+  def initialize
+    super()
+    @tree = RBTree.new
+  end
+
+  def query(word)
+    synchronize do
+      @tree.bound([word, 0, ''], [word + "\0", 0, '']).collect {|k, v| k[2]}
+    end
+  end
+
+  def delete(fname, mtime, src)
+    synchronize do
+      each_tree_key(fname, mtime, src) do |key|
+        @tree.delete(key)
+      end
+    end
+  end
+
+  def push(fname, mtime, src)
+    synchronize do
+      each_tree_key(fname, mtime, src) do |key|
+        @tree[key] = true
+      end
+    end
+  end
+
+  def intern(word)
+    k, v = @tree.lower_bound([word, 0, ''])
+    return k[0] if k && k[0] == word
+    word
+  end
+
+  def each_tree_key(fname, mtime, src)
+    NKF.nkf('-w', src).scan(/\w+/m).uniq.each do |word|
+      yield([intern(word), mtime.to_i, fname])
+    end
+  end
+end
+
+if __FILE__ == $0
+  indexer ||= Indexer.new(0)
+  Thread.new do
+    indexer.update_dict
+  end
+  
+  while line = gets
+    ary = indexer.dict.query(line.chomp)
+    pp ary
+    pp ary.size
+  end
+end
+