rhosync 2.0.0.beta1

data/lib/rhosync/store.rb

@@ -0,0 +1,199 @@
+module Rhosync
+  class Store
+    RESERVED_ATTRIB_NAMES = ["attrib_type", "id"] unless defined? RESERVED_ATTRIB_NAMES
+    @@db = nil
+    
+    class << self
+      def db; @@db || @@db = _get_redis end
+      
+      def db=(server=nil)
+        @@db = _get_redis(server)
+      end
+      
+      def create(server=nil)
+        @@db ||= _get_redis(server)
+        raise "Error connecting to Redis store." unless @@db and (@@db.is_a?(Redis) or @@db.is_a?(Redis::Client))
+      end
+  
+      # Adds set with given data, replaces existing set
+      # if it exists or appends data to the existing set
+      # if append flag set to true
+      def put_data(dockey,data={},append=false)
+        if dockey and data
+          flash_data(dockey) unless append
+          # Inserts a hash or array
+          if data.is_a?(Hash)
+            @@db.pipelined do |pipeline|
+              data.each do |key,value|
+                value.each do |attrib,value|
+                  unless _is_reserved?(attrib,value)
+                    pipeline.sadd(dockey,setelement(key,attrib,value))
+                  end
+                end
+              end
+            end
+          else
+            @@db.pipelined do |pipeline|
+              data.each do |value|
+                pipeline.sadd(dockey,value)
+              end
+            end
+          end
+        end
+        true
+      end
+    
+      # Adds a simple key/value pair
+      def put_value(dockey,value)
+        if dockey
+          @@db.del(dockey)
+          @@db.set(dockey,value.to_s) if value
+        end
+      end
+    
+      # Retrieves value for a given key
+      def get_value(dockey)
+          @@db.get(dockey) if dockey
+      end
+  
+      # Retrieves set for given dockey,source,user
+      def get_data(dockey,type=Hash)
+        res = type == Hash ? {} : []
+        if dockey
+          @@db.smembers(dockey).each do |element|
+            if type == Hash
+              key,attrib,value = getelement(element)
+              res[key] = {} unless res[key]
+              res[key].merge!({attrib => value})
+            else
+              res << element
+            end
+          end
+          res
+        end
+      end
+  
+      # Retrieves diff data hash between two sets
+      def get_diff_data(src_dockey,dst_dockey,p_size=nil)
+        res = {}
+        if src_dockey and dst_dockey
+          @@db.sdiff(dst_dockey,src_dockey).each do |element|
+            key,attrib,value = getelement(element)
+            res[key] = {} unless res[key]
+            res[key].merge!({attrib => value})
+          end
+        end
+        if p_size
+          diff = {}
+          page_size = p_size
+          res.each do |key,item|
+            diff[key] = item
+            page_size -= 1
+            break if page_size <= 0         
+          end
+          [diff,res.size]
+        else  
+          [res,res.size]
+        end
+      end
+
+      # Deletes data from a given doctype,source,user
+      def delete_data(dockey,data={})
+        if dockey and data
+          @@db.pipelined do |pipeline|
+            data.each do |key,value|
+              value.each do |attrib,val|
+                pipeline.srem(dockey,setelement(key,attrib,val))
+              end
+            end
+          end
+        end
+        true
+      end
+    
+      # Deletes all keys matching a given mask
+      def flash_data(keymask)
+        @@db.keys(keymask).each do |key|
+          @@db.del(key)
+        end
+      end
+    
+      # Returns array of keys matching a given keymask
+      def get_keys(keymask)
+        @@db.keys(keymask)
+      end
+    
+      # Returns true if given item is a member of the given set
+      def ismember?(setkey,item)
+        @@db.sismember(setkey,item)
+      end
+      
+      # Lock a given key and release when provided block is finished
+      def lock(dockey,timeout=0)
+        m_lock = get_lock(dockey,timeout)
+        res = yield
+        release_lock(dockey,m_lock)
+        res
+      end
+    
+      def get_lock(dockey,timeout=0)
+        lock_key = _lock_key(dockey)
+        current_time = Time.now.to_i
+        if not @@db.setnx(lock_key,current_time+timeout+1)
+          loop do
+            if @@db.get(lock_key).to_i <= current_time and 
+                @@db.getset(lock_key,current_time+timeout+1).to_i <= current_time
+              break
+            end
+            sleep(1)
+            current_time = Time.now.to_i
+          end
+        end
+        current_time+timeout+1
+      end
+      
+      # Due to redis bug #140, setnx always returns true so this doesn't work
+      # def get_lock(dockey,timeout=0)
+      #   lock_key = _lock_key(dockey)
+      #   until @@db.setnx(lock_key,1) do 
+      #     sleep(1) 
+      #   end
+      #   @@db.expire(lock_key,timeout+1)
+      #   Time.now.to_i+timeout+1
+      # end
+      
+      def release_lock(dockey,lock)
+        @@db.del(_lock_key(dockey)) if (lock >= Time.now.to_i)
+      end
+      
+      # Create a copy of srckey in dstkey
+      def clone(srckey,dstkey)
+        @@db.sdiffstore(dstkey,srckey,'')
+      end
+      
+      # Rename srckey to dstkey
+      def rename(srckey,dstkey)
+        @@db.rename(srckey,dstkey) if @@db.exists(srckey)
+      end
+      
+      private
+      def _get_redis(server=nil)
+        if server and server.is_a?(String)
+          host,port,db,password = server.split(':')
+          Redis.new(:thread_safe => true, :host => host,
+            :port => port, :db => db, :password => password)
+        else
+          Redis.new(:thread_safe => true)
+        end
+      end
+      
+      def _lock_key(dockey)
+        "#{dockey}:lock"
+      end
+          
+      def _is_reserved?(attrib,value) #:nodoc:
+        RESERVED_ATTRIB_NAMES.include? attrib
+      end
+    end
+  end
+end  

data/lib/rhosync/tasks.rb

@@ -0,0 +1,151 @@
+require 'json'
+require 'mechanize'
+require 'zip/zip'
+require 'uri'
+require File.join(File.dirname(__FILE__),'console','rhosync_api')
+
+module Rhosync
+  module TaskHelper
+    def post(path,params)
+      req = Net::HTTP.new($host,$port)
+      resp = req.post(path, params.to_json, 'Content-Type' => 'application/json')
+      print_resp(resp, resp.is_a?(Net::HTTPSuccess) ? true : false)
+    end
+
+    def print_resp(resp,success=true)
+      if success
+        puts "=> OK" 
+      else
+        puts "=> FAILED"
+      end
+      puts "=> " + resp.body if resp and resp.body and resp.body.length > 0
+    end
+
+    def archive(path)
+      File.join(path,File.basename(path))+'.zip'
+    end
+
+    def ask(msg)
+      print msg
+      STDIN.gets.chomp
+    end
+    
+    def load_settings(file)
+      begin
+        $settings = YAML.load_file(file)
+      rescue Exception => e
+        puts "Error opening settings file #{file}: #{e}."
+        puts e.backtrace.join("\n")
+        raise e
+      end
+    end
+    
+    def rhosync_socket
+      "/tmp/rhosync.dtach"
+    end
+    
+    def rhosync_pid
+      "/tmp/rhosync.pid"
+    end
+  end
+end
+
+namespace :rhosync do
+  include Rhosync::TaskHelper
+  include RhosyncApi
+  
+  task :config do
+    $settings = load_settings(File.join(ENV['PWD'],'settings','settings.yml'))
+    env = (ENV['RHO_ENV'] || :development).to_sym  
+    uri = URI.parse($settings[env][:syncserver])
+    $url = "#{uri.scheme}://#{uri.host}"
+    $url = "#{$url}:#{uri.port}" if uri.port && uri.port != 80
+    $host = uri.host
+    $port = uri.port
+    $agent = Mechanize.new
+    $appname = $settings[env][:syncserver].split('/').last
+    $token_file = File.join(ENV['HOME'],'.rhosync_token')
+    $token = File.read($token_file) if File.exist?($token_file)
+  end
+  
+  desc "Reset the rhosync database (you will need to run rhosync:get_api_token afterwards)"
+  task :reset => :config do
+    RhosyncApi.reset($url,$token)
+  end
+  
+  desc "Fetches current api token from rhosync"
+  task :get_token => :config do
+    login = ask "admin login: "
+    password = ask "admin password: "
+    $token = RhosyncApi.get_token($url,login,password)
+    File.open($token_file,'w') {|f| f.write $token}
+    puts "Token is saved in: #{$token_file}"
+  end
+  
+  desc "Clean rhosync, get token, and create new user"
+  task :clean_start => [:get_token, :reset, :get_token, :create_user]
+  
+  desc "Alias for `rake rhosync:stop; rake rhosync:start`"
+  task :restart => [:stop, :start]
+  
+  desc "Creates and subscribes user for application in rhosync"
+  task :create_user => :config do
+    login = ask "new user login: "
+    password = ask "new user password: "
+    RhosyncApi.create_user($url,$appname,$token,login,password)
+  end
+  
+  desc "Deletes the user from rhosync"
+  task :delete_user => :config do
+    login = ask "user to delete: "
+    RhosyncApi.delete_user($url,$appname,$token,login)
+  end
+  
+  # desc "Updates an existing user in rhosync"
+  # task :update_user => :config do
+  #   login = ask "login: "
+  #   password = ask "password: "
+  #   new_password = ask "new password: "
+  #   post("/api/update_user", {:app_name => $appname, :api_token => $token,
+  #     :login => login, :password => password, :attributes => {:new_password => new_password}})
+  # end
+  
+  # desc "Reset source refresh time"
+  # task :reset_refresh_time => :config do
+  #   user = ask "user: "
+  #   source_name = ask "source name: "
+  #   post("/api/set_refresh_time", {:api_token => $token, :app_name => $appname,
+  #     :user_name => user, :source_name => source_name})
+  # end
+  
+  desc "Run rhosync source adapter specs"
+  task :spec do
+    files = File.join($app_basedir,'rhosync/spec/sources/*_spec.rb')
+    Spec::Rake::SpecTask.new('rhosync:spec') do |t|
+      t.spec_files = FileList[files]
+      t.spec_opts = %w(-fs --color)
+      t.rcov = true
+      t.rcov_opts = ['--exclude', 'spec/*,gems/*']
+    end
+  end
+  
+  desc "Start rhosync server"
+  task :start do
+    puts 'Detach with Ctrl+\  Re-attach with rake rhosync:attach'
+    sleep 1
+    command = "dtach -A #{rhosync_socket} rackup config.ru -P #{rhosync_pid}"
+    sh command
+  end
+  
+  desc "Stop rhosync server"
+  task :stop do
+    sh "cat #{rhosync_pid} | xargs kill -3"
+  end
+  
+  desc "Attach to rhosync console"
+  task :attach do
+    sh "dtach -a #{rhosync_socket}"
+  end
+end
+
+load File.join(File.dirname(__FILE__),'..','..','tasks','redis.rake')

data/lib/rhosync/user.rb

@@ -0,0 +1,83 @@
+require 'digest/sha1'
+
+module Rhosync
+  # Inspired by sinatra-authentication
+  # Password uses simple sha1 digest for hashing
+  class User < Model
+    field :login,:string
+    field :email,:string
+    field :salt,:string
+    field :hashed_password,:string
+    set   :clients, :string
+    field :admin, :int
+    field :token_id, :string
+    
+    class << self
+      def create(fields={})
+        fields[:id] = fields[:login]
+        super(fields)
+      end
+    
+      def authenticate(login,password)
+        return unless is_exist?(login)
+        current_user = load(login)
+        return if current_user.nil?
+        return current_user if User.encrypt(password, current_user.salt) == current_user.hashed_password
+      end
+    end
+    
+    def new_password=(pass)
+      self.password=(pass)
+    end
+    
+    def password=(pass)
+      @password = pass
+      self.salt = User.random_string(10) if !self.salt
+      self.hashed_password = User.encrypt(@password, self.salt)
+    end
+    
+    def delete
+      clients.members.each do |client_id|
+        Client.load(client_id,{:source_name => '*'}).delete
+      end
+      self.token.delete if self.token
+      super
+    end
+    
+    def create_token
+      if self.token_id && ApiToken.is_exist?(self.token_id)
+        self.token.delete 
+      end
+      self.token_id = ApiToken.create(:user_id => self.login).id
+    end
+    
+    def token
+      ApiToken.load(self.token_id)
+    end
+    
+    def token=(value)
+      if self.token_id && ApiToken.is_exist?(self.token_id)
+        self.token.delete 
+      end
+      self.token_id = ApiToken.create(:user_id => self.login, :value => value).id
+    end
+    
+    def update(fields)
+      fields.each do |key,value|
+        self.send("#{key.to_sym}=", value) unless key == 'login'
+      end  
+    end
+      
+    protected
+    def self.encrypt(pass, salt)
+      Digest::SHA1.hexdigest(pass+salt)
+    end
+
+    def self.random_string(len)
+      chars = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a
+      newpass = ""
+      1.upto(len) { |i| newpass << chars[rand(chars.size-1)] }
+      return newpass
+    end
+  end
+end

data/lib/rhosync/version.rb

@@ -0,0 +1,3 @@
+module Rhosync
+  VERSION = '2.0.0.beta1'
+end

data/lib/rhosync.rb

@@ -0,0 +1,251 @@
+require 'redis'
+require 'json'
+require 'base64'
+require 'zip/zip'
+require 'yaml'
+require 'rhosync/license'
+require 'rhosync/version'
+require 'rhosync/document'
+require 'rhosync/lock_ops'
+require 'rhosync/model'
+require 'rhosync/source'
+require 'rhosync/user'
+require 'rhosync/api_token'
+require 'rhosync/app'
+require 'rhosync/store'
+require 'rhosync/client'
+require 'rhosync/read_state'
+require 'rhosync/client_sync'
+require 'rhosync/source_adapter'
+require 'rhosync/source_sync'
+require 'rhosync/indifferent_access'
+require 'rhosync/jobs/source_job'
+require 'rhosync/jobs/ping_job'
+require 'rhosync/bulk_data'
+  
+# Various module utilities for the store
+module Rhosync
+  APP_NAME = 'application' unless defined? APP_NAME
+  
+  class InvalidArgumentError < RuntimeError; end
+  class RhosyncServerError < RuntimeError; end
+  extend self
+    
+  class << self
+    attr_accessor :base_directory, :app_directory, :data_directory, 
+      :vendor_directory, :blackberry_bulk_sync, :redis, :environment,
+      :log_disabled, :license
+  end
+  
+  ### Begin Rhosync setup methods  
+  # Server hook to initialize Rhosync
+  def bootstrap(basedir)
+    config = get_config(basedir)
+    #Load environment
+    environment = (ENV['RHO_ENV'] || :development).to_sym     
+    # Initialize Rhosync and Resque
+    Rhosync.base_directory = basedir    
+    Rhosync.app_directory = get_setting(config,environment,:app_directory)
+    Rhosync.data_directory = get_setting(config,environment,:data_directory)
+    Rhosync.vendor_directory = get_setting(config,environment,:vendor_directory)
+    Rhosync.blackberry_bulk_sync = get_setting(config,environment,:blackberry_bulk_sync,false)
+    Rhosync.redis = get_setting(config,environment,:redis,false)
+    Rhosync.log_disabled = get_setting(config,environment,:log_disabled,false)
+    Rhosync.environment = environment    
+    yield self if block_given?
+    Store.create(Rhosync.redis)
+    Resque.redis = Store.db
+    Rhosync.base_directory ||= File.join(File.dirname(__FILE__),'..')
+    Rhosync.app_directory ||= Rhosync.base_directory
+    Rhosync.data_directory ||= File.join(Rhosync.base_directory,'data')
+    Rhosync.vendor_directory ||= File.join(Rhosync.base_directory,'vendor')
+    Rhosync.blackberry_bulk_sync ||= false
+    Rhosync.log_disabled ||= false
+    Rhosync.license = License.new
+    
+    check_and_add(File.join(Rhosync.app_directory,'sources'))
+    start_app(config)
+    create_admin_user
+    check_hsql_lib! if Rhosync.blackberry_bulk_sync
+  end
+  
+  def start_app(config)
+    if config and config[Rhosync.environment]
+      app = nil
+      app_name = APP_NAME
+      if App.is_exist?(app_name)
+        app = App.load(app_name)
+      else
+        app = App.create(:name => app_name)
+      end
+      sources = config[:sources] || []
+      sources.each do |source_name,fields|
+        if Source.is_exist?(source_name)  
+          s = Source.load(source_name,{:app_id => app.name,:user_id => '*'})
+          s.update(fields)
+        else  
+          fields[:name] = source_name
+          Source.create(fields,{:app_id => app.name})
+        end
+        unless app.sources.members.include?(source_name)
+          app.sources << source_name
+        end
+        # load ruby file for source adapter to re-load class
+        load under_score(source_name+'.rb')
+      end
+    end
+  end
+  
+  # Generate admin user on first load
+  def create_admin_user
+    unless User.is_exist?('admin')
+      admin = User.create({:login => 'admin', :admin => 1})
+      admin.password = ''
+      admin.create_token
+    end
+  end
+
+  # Add path to load_path unless it has been added already
+  def check_and_add(path)
+    $:.unshift path unless $:.include?(path) 
+  end
+
+  def get_config(basedir)
+    # Load settings
+    settings_file = File.join(basedir,'settings','settings.yml') if basedir
+    config = YAML.load_file(settings_file) if settings_file and File.exist?(settings_file)
+  end
+  ### End Rhosync setup methods  
+  
+  
+  def check_default_secret!(secret)
+    if secret == '<changeme>'                        
+      log "*"*60+"\n\n"
+      log "WARNING: Change the session secret in config.ru from <changeme> to something secure."
+      log "  i.e. running `rake secret` in a rails app will generate a secret you could use.\n\n"
+      log "*"*60
+    end
+  end
+
+  # Serializes oav to set element
+  def setelement(obj,attrib,value)
+    "#{obj}:#{attrib}:#{Base64.encode64(value.to_s)}"
+  end
+
+  # De-serializes oav from set element
+  def getelement(element)
+    res = element.split(':')
+    [res[0], res[1], Base64.decode64(res[2].to_s)]
+  end
+
+  # Get random UUID string
+  def get_random_uuid
+    UUIDTools::UUID.random_create.to_s.gsub(/\-/,'')
+  end
+
+  # Generates new token (64-bit integer) based on # of 
+  # microseconds since Jan 1 2009
+  def get_token
+    ((Time.now.to_f - Time.mktime(2009,"jan",1,0,0,0,0).to_f) * 10**6).to_i
+  end
+
+  # Computes token for a single client request
+  def compute_token(doc_key)
+    token = get_token
+    Store.put_value(doc_key,token)
+    token.to_s
+  end
+
+  # Returns require-friendly filename for a class
+  def under_score(camel_cased_word)
+    camel_cased_word.to_s.gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    tr("-", "_").
+    downcase
+  end
+
+  # Taken from rails inflector
+  def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+    if first_letter_in_uppercase
+      lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+    end
+  end
+
+  def check_hsql_lib!
+    unless File.exists?(File.join(Rhosync.vendor_directory,'hsqldata.jar'))
+      log "*"*60
+      log ""
+      log "WARNING: Missing vendor/hsqldata.jar, please install it for BlackBerry bulk sync support."
+      log ""
+      log "*"*60
+    end
+  end
+
+  def unzip_file(file_dir,params)
+    uploaded_file = File.join(file_dir, params[:filename])
+    begin
+      File.open(uploaded_file, 'wb') do |file|
+        file.write(params[:tempfile].read)
+      end
+      Zip::ZipFile.open(uploaded_file) do |zip_file|
+        zip_file.each do |f|
+          f_path = File.join(file_dir,f.name)
+          FileUtils.mkdir_p(File.dirname(f_path))
+          zip_file.extract(f, f_path) { true }
+        end
+      end
+    rescue Exception => e
+      log "Failed to unzip `#{uploaded_file}`"
+      raise e
+    ensure
+      FileUtils.rm_f(uploaded_file)
+    end
+  end
+
+  def lap_timer(msg,start)
+    duration = timenow - start
+    log "#{msg}: #{duration}"
+    timenow
+  end
+
+  def start_timer(msg='starting')
+    log "#{msg}"
+    timenow
+  end
+
+  def timenow
+    (Time.now.to_f * 1000)
+  end
+  
+  def log(*args)
+    now = Time.now.strftime('%I:%M:%S %p %Y-%m-%d')
+    puts "[#{now}] #{args.join}" unless Rhosync.log_disabled
+  end
+  
+  # Base rhosync application class
+  class Base
+    # Add everything in vendor to load path
+    # TODO: Integrate with 3rd party dependency management
+    def self.initializer(path=nil)
+      Dir["vendor/*"].each do |dir|
+        $:.unshift File.join(dir,'lib')
+      end
+      require 'rhosync'
+      require 'rhosync/server'
+      # Bootstrap Rhosync system
+      Rhosync.bootstrap(path || ENV['PWD'])
+    end
+    
+    def self.store_blob(obj,field_name,blob)
+      blob[:tempfile].path if blob[:tempfile]
+    end
+  end
+  
+  protected
+  def get_setting(config,environment,setting,default=nil)
+    res = nil
+    res = config[environment][setting] if config and environment 
+    res || default
+  end
+end