pastehub 0.2.6

data/lib/pastehub/store.rb

@@ -0,0 +1,83 @@
+#
+# store.rb - PasteHub's storage sync library
+#  
+#   Copyright (c) 2009-2011  Kiyoka Nishiyama  <kiyoka@sumibi.org>
+#   
+#   Redistribution and use in source and binary forms, with or without
+#   modification, are permitted provided that the following conditions
+#   are met:
+#   
+#   1. Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#  
+#   2. Redistributions in binary form must reproduce the above copyright
+#      notice, this list of conditions and the following disclaimer in the
+#      documentation and/or other materials provided with the distribution.
+#  
+#   3. Neither the name of the authors nor the names of its contributors
+#      may be used to endorse or promote products derived from this
+#      software without specific prior written permission.
+#  
+#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+#   TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+#   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+#   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+#   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+#   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#  
+#
+
+module PasteHub
+
+  class LocalStore
+    def initialize( username, reader = false )
+      @db = PasteHub::LocalDB.new( PasteHub::Config.instance.localDbPath )
+      @db.open( username, reader )
+    end
+
+    def getList
+      @db.getList()
+    end
+
+    def getServerList
+      @db.getServerList()
+    end
+
+    def top
+      lst = @db.getList( 1 )
+      if 0 < lst.size
+        [ lst.first, @db.getValue( lst.first ) ]
+      else
+        [ nil, nil ]
+      end
+    end
+
+    def getValue( key )
+      @db.getValue( key.dup )
+    end
+
+    def latest
+      [ @db.getValue( PasteHub::SERVER_DATE_KEY ).to_s,
+        @db.getValue( PasteHub::LOCAL_DATE_KEY  ).to_s ]
+    end
+
+    def insertValue( key, value )
+      @db.insertValue( key, value )
+    end
+
+    def close
+      @db.close
+    end
+
+    def clear
+      @db.clear
+    end
+
+  end
+
+end

data/lib/pastehub/util.rb

@@ -0,0 +1,153 @@
+require 'digest'
+require 'date'
+require 'set'
+
+module PasteHub
+  require 'rubygems'
+  begin
+    require 'highline'
+    USE_HIGHLINE = true
+  rescue LoadError
+    USE_HIGHLINE = false
+  end
+
+  class Util
+    def initialize()
+    end
+
+    # return message digest for str.
+    def digest( str )
+      Digest::SHA1.hexdigest( str )
+    end
+
+    # return the currentTime in Unixtime
+    def currentTime( )
+      dt = Time.new.gmtime.to_datetime()
+      currentDate = dt.strftime( "%s" ) + "=" + dt.strftime( "%F.%H:%M:%S" )
+      currentDate
+    end
+
+    def currentSeconds( )
+      self.key_seconds( self.currentTime() )
+    end
+
+    def _splitKey( key )
+      key.split( /[=]/ )
+    end
+
+    def key_seconds( key )
+      arr = _splitKey( key )
+      if 1 < arr.size
+        arr[0].to_i
+      else
+        nil
+      end
+    end
+
+    def key_timestamp( key )
+      arr = _splitKey( key )
+      if 1 < arr.size
+        arr[1]
+      else
+        nil
+      end
+    end
+
+    def key_digest( key )
+      arr = _splitKey( key )
+      if 2 < arr.size
+        arr[2]
+      else
+        nil
+      end
+    end
+
+    def diffList( list1, list2 )
+      set1 = Set.new
+      set2 = Set.new
+      list1.each { |e| set1.add(e) }
+      list2.each { |e| set2.add(e) }
+      set1.difference( set2 ).to_a
+    end
+
+    # Same as Gauche's take* function
+    def takeList( list1, num )
+      if ( num < 0 )
+        list1
+      elsif num <= list1.size
+        list1[ 0 ... num ]
+      else
+        list1
+      end
+    end
+
+    # Same as Gauche's drop* function
+    def dropList( list1, num )
+      if num < 0
+        list1
+      elsif num <= list1.size
+        list1[ num .. list1.size ]
+      else
+        []
+      end
+    end
+
+    def say( message )
+      if USE_HIGHLINE
+        HighLine.new.say( message ) {|q|
+          q.readline = true
+        }
+      else
+        puts "#{message}"
+      end
+    end
+
+    def inputText( label )
+      if USE_HIGHLINE
+        HighLine.new.ask( label )  {|q|
+          q.readline = true
+        }
+      else
+        print "#{label}"
+        return gets.chomp
+      end
+    end
+    
+    def inputPassword( label )
+      if USE_HIGHLINE
+        HighLine.new.ask( label )  {|q|
+          q.readline = true
+          q.echo = '*'
+        }
+      else
+        print "#{label}"
+        return gets.chomp
+      end
+    end
+
+    # input utility
+    def inputPasswordTwice( message, firstLabel, secondLabel )
+      required_password_chars = 6
+
+      STDERR.puts( message )
+      3.times { |n|
+        firstStr = nil
+        while not firstStr
+          firstStr  = inputPassword(firstLabel)
+          if required_password_chars > firstStr.size()
+            STDERR.puts( "you must input #{required_password_chars} or more characters." )
+            firstStr = nil
+          elsif firstStr.match( /[ \t]/i )
+            STDERR.puts( "you must not use white space characters." )
+            firstStr = nil
+          end
+        end
+        secondStr = inputPassword(secondLabel)
+        if firstStr == secondStr
+          return firstStr
+        end
+      }
+      return nil
+    end
+  end
+end

data/server/masterdb.rb

@@ -0,0 +1,141 @@
+require 'vertx'
+require 'json/pure'
+require 'cgi'
+require 'date'
+require 'memcache'
+
+$LOAD_PATH.push( File.dirname(__FILE__) + "/../lib" )
+require 'pastehub'
+require 'pastehub/log'
+PasteHub::Config.instance.loadServer
+
+# display config info
+ins = PasteHub::Config.instance
+printf( "Use AWS:                 %s\n", ins.aws )
+printf( "Domain:                  %s\n", ins.domain )
+printf( "Dynamo   Endpoint:       %s\n", ins.dynamoEp )
+printf( "Memcache Endpoint:       %s\n", ins.memcacheEp )
+
+# initialize master database
+require 'pastehub/masterdb'
+
+
+# setup user table for Fake DynamoDB
+users = PasteHub::Users.new( )
+if not ins.aws
+  open( "/var/pastehub/users.tsv", "r" ) {|f|
+    f.readlines.each { |line|
+      pair = line.chomp.split( /[\t ]+/ )
+      printf( "Added local user table:  %s\n", pair[0] )
+      users.addUser( pair[0], pair[1] )
+    }
+  }
+end
+
+
+notifyHash = Memcache.new( :server => PasteHub::Config.instance.memcacheEp )
+
+masterdb_server = Vertx::HttpServer.new
+masterdb_server.request_handler do |req|
+
+  req.body_handler do |body|
+    util = PasteHub::Util.new
+    auth = PasteHub::AuthForServer.new( users )
+    ret = auth.invoke( req.headers, util.currentSeconds() )
+    username = ret[1]
+
+    log = PasteHub::Log.new( :api => req.path, :user => username )
+    if ret[0]
+      log.info( "connected" )
+    else
+      log.error( 'Auth failure:' + ret[2].to_s, { :reason => ret[2].to_s } )
+      req.response.status_code = 403
+      req.response.status_message = "Authorization failure."
+      req.response.end
+      return
+    end
+
+    entries = PasteHub::Entries.new( username )
+    users   = PasteHub::Users.new( )
+
+    case req.path
+    when "/authTest"
+      req.response.end
+
+    when "/putValue"
+      data = body.to_s.dup
+      digest = util.digest( data )
+
+      # update db
+      key = req.headers[ 'X-Pastehub-Key' ].dup
+      puts "[#{username}]:putValue: key=[#{key}] "
+
+      # client have no specified key
+      if "_" == key
+        key = util.currentTime( ) + "=" + digest
+      end
+
+      # data duplicate check
+      insertFlag = true
+      prevKey = notifyHash.get( username )
+      if prevKey
+        if util.key_digest( prevKey ) == digest
+          log.info( "canceled because data is duplicate. ", { :key => key } )
+          insertFlag = false
+          key = prevKey
+        end
+      end
+
+      if insertFlag
+        Vertx.set_timer(1000) do
+          log.info( "START: delayed job." )
+          # update db
+          log.info( "insert", { :key => key } )
+          entries.insertValue( key, data )
+          users.touch( username )
+          # notify to client
+          notifyHash.set( username, key, PasteHub::Config.instance.keyCacheTime )
+
+          # remove Last entry
+          arr = entries.getList( )
+          if PasteHub::Config.instance.listItems < arr.size
+            entries.deleteValue( arr[arr.size-1] )
+          end
+          log.info( "END:   delayed job", { :entries => arr.size } )
+        end
+      end
+      req.response.end( key )
+
+
+    when "/getList"
+      limit = req.headers[ 'X-Pastehub-Limit' ]
+      if limit
+        str = entries.getList( ).take( limit.to_i ).join( "\n" )
+      else
+        str = entries.getList( ).join( "\n" )
+      end
+      log.info( "getList", { :limit => limit, :entries => entries.getList( ).size } )
+      puts str
+      req.response.end( str )
+
+    when "/getValue"
+      k = body.to_s.chomp
+      if 0 < k.size
+        str = entries.getValue( k.dup )
+      else
+        str = ""
+      end
+      log.info( "getValue", { :key => k } )
+      puts "[#{username}]:getValue:" + k
+      req.response.end( str )
+
+    else
+      mes = "Error: Unknown API #{req.path}"
+      log.error( mes )
+      req.response.status_code = 400
+      req.response.status_message = mes
+      req.response.end
+
+    end
+  end
+end.listen(8000)

data/server/notifier.rb

@@ -0,0 +1,89 @@
+require 'vertx'
+require 'cgi'
+require 'memcache'
+require 'pp'
+$LOAD_PATH.push( File.dirname(__FILE__) + "/../lib" )
+require 'pastehub'
+require 'pastehub/log'
+PasteHub::Config.instance.loadServer
+
+# display config info
+ins = PasteHub::Config.instance
+printf( "Use AWS:                 %s\n", ins.aws )
+printf( "Domain:                  %s\n", ins.domain )
+printf( "Dynamo   Endpoint:       %s\n", ins.dynamoEp )
+printf( "Memcache Endpoint:       %s\n", ins.memcacheEp )
+printf( "ssl keystore file:       %s\n", ins.keystore )
+
+# initialize master database
+require 'pastehub/masterdb'
+
+# setup user table for Fake DynamoDB
+users = PasteHub::Users.new( )
+if not ins.aws
+  open( "/var/pastehub/users.tsv", "r" ) {|f|
+    f.readlines.each { |line|
+      pair = line.chomp.split( /[\t ]+/ )
+      printf( "Added local user table:  %s\n", pair[0] )
+      users.addUser( pair[0], pair[1] )
+    }
+  }
+end
+
+
+INTERVAL    = 0.5
+POLLING_SEC = 60
+
+notifyHash = Memcache.new( :server => PasteHub::Config.instance.memcacheEp )
+
+def notify( res, str )
+  res.write_str( "#{str}\n"  )
+end
+
+notifier = Vertx::HttpServer.new
+if ins.keystore
+  notifier.ssl = true
+  notifier.key_store_path     = ins.keystore
+  notifier.key_store_password = ins.keystorePassword
+end
+
+notifier.request_handler do |req|
+  util = PasteHub::Util.new
+  auth = PasteHub::AuthForServer.new( users )
+
+  ret = auth.invoke( req.headers, util.currentSeconds() )
+  username = ret[1]
+  # Now send back a response
+  req.response.chunked = true
+
+  log = PasteHub::Log.new( :api => 'notifier', :user => username )
+  if ret[0]
+    log.info( "connected" )
+  else
+    log.error( 'Auth failure:' + ret[2].to_s, { :reason => ret[2].to_s } )
+    req.response.status_code = 403
+    req.response.status_message = "Authorization failure."
+    req.response.end
+    return
+  end
+
+  got         = nil
+  timer_count = 0
+
+  Vertx::set_periodic (1000 * INTERVAL) { |timer_id|
+    timer_count += INTERVAL
+    if notifyHash[ username ]
+      if got != notifyHash[ username ]
+        got = notifyHash[ username ]
+        log.info( 'notify', { :notify => got } )
+        notify( req.response, got )
+      end
+    end
+
+    if ( POLLING_SEC < timer_count )
+      log.info( 'timeout', { :notify => nil } )
+      req.response.end
+      Vertx::cancel_timer(timer_id)
+    end
+  }
+end.listen(8001)