rhosync 2.0.0.beta1

data/lib/rhosync/api/list_clients.rb

@@ -0,0 +1,3 @@
+Rhosync::Server.api :list_clients do |params,user|
+  User.load(params[:user_id]).clients.members.to_json
+end

data/lib/rhosync/api/list_source_docs.rb

@@ -0,0 +1,10 @@
+Rhosync::Server.api :list_source_docs do |params,user|
+  res = {}
+  s = Source.load(params[:source_id], {:app_id => APP_NAME,:user_id => params[:user_id]})
+  [:md,:md_size,:md_copy,:errors].each do |doc|
+    db_key = s.docname(doc)
+    res.merge!(doc => db_key)
+  end
+  res.to_json
+end
+

data/lib/rhosync/api/list_sources.rb

@@ -0,0 +1,15 @@
+Rhosync::Server.api :list_sources do |params,user|
+  sources = App.load(APP_NAME).sources.members
+  if params[:partition_type].nil? or params[:partition_type] == 'all'
+    sources.to_json 
+  else
+    res = []
+    sources.each do |name|
+      s = Source.load(name,{:app_id => APP_NAME,:user_id => '*'})
+      if s.partition_type and s.partition_type == params[:partition_type]
+        res << name 
+      end
+    end  
+    res.to_json
+  end  
+end

data/lib/rhosync/api/list_users.rb

@@ -0,0 +1,3 @@
+Rhosync::Server.api :list_users do |params,user|
+  App.load(APP_NAME).users.members.to_json
+end

data/lib/rhosync/api/ping.rb

@@ -0,0 +1,7 @@
+Rhosync::Server.api :ping do |params,user|
+  if params[:async]
+    PingJob.enqueue(params)
+  else
+    PingJob.perform(params)
+  end
+end

data/lib/rhosync/api/push_deletes.rb

@@ -0,0 +1,6 @@
+Rhosync::Server.api :push_deletes do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  source_sync = SourceSync.new(source)
+  source_sync.push_deletes(params[:objects])
+  'done'
+end

data/lib/rhosync/api/push_objects.rb

@@ -0,0 +1,6 @@
+Rhosync::Server.api :push_objects do |params,user|
+  source = Source.load(params[:source_id],{:app_id=>APP_NAME,:user_id=>params[:user_id]})
+  source_sync = SourceSync.new(source)
+  source_sync.push_objects(params[:objects])
+  'done'
+end

data/lib/rhosync/api/reset.rb

@@ -0,0 +1,10 @@
+Rhosync::Server.api :reset do |params,user|
+  Store.db.flushdb
+  app_klass = Object.const_get(camelize(APP_NAME))
+  if app_klass.singleton_methods.include?("initializer")
+    app_klass.send :initializer, Rhosync.base_directory
+  end
+  # restoring previous token value after flushdb
+  user.token = params[:api_token]
+  "DB reset"
+end

data/lib/rhosync/api/set_db_doc.rb

@@ -0,0 +1,8 @@
+Rhosync::Server.api :set_db_doc do |params,user|
+  if params[:data_type] and params[:data_type] == 'string'
+    Store.put_value(params[:doc],params[:data])
+  else
+    Store.put_data(params[:doc],params[:data])
+  end  
+  ''
+end

data/lib/rhosync/api/set_refresh_time.rb

@@ -0,0 +1,8 @@
+Rhosync::Server.api :set_refresh_time do |params,user|
+  source = Source.load(params[:source_name],
+    {:app_id => APP_NAME, :user_id => params[:user_name]})
+  source.poll_interval = params[:poll_interval] if params[:poll_interval]
+  params[:refresh_time] ||= 0
+  source.read_state.refresh_time = Time.now.to_i + params[:refresh_time].to_i
+  ''
+end

data/lib/rhosync/api/update_user.rb

@@ -0,0 +1,4 @@
+Rhosync::Server.api :update_user do |params,user|
+  user.update(params[:attributes])
+  "User updated"
+end

data/lib/rhosync/api/upload_file.rb

@@ -0,0 +1,4 @@
+Rhosync::Server.api :upload_file do |params,user|
+  unzip_file(Rhosync.app_directory,params[:upload_file]) if File.exists?(Rhosync.app_directory)
+  "File uploaded"
+end

data/lib/rhosync/api_token.rb

@@ -0,0 +1,19 @@
+require 'uuidtools'
+module Rhosync
+  class ApiToken < Model
+    field :value,:string
+    field :user_id,:string
+    validates_presence_of :user_id
+    
+    def self.create(fields)
+      fields[:value] = fields[:value] || get_random_uuid
+      fields[:id] = fields[:value]
+      object = super(fields)
+    end
+    
+    def user
+      @user ||= User.load(self.user_id)
+    end
+  end
+end
+    

data/lib/rhosync/app.rb

@@ -0,0 +1,69 @@
+module Rhosync
+  class App < Model
+    field :name, :string
+    set   :users, :string
+    set   :sources, :string
+    attr_reader :delegate
+    validates_presence_of :name
+    
+    class << self
+      def create(fields={})
+        fields[:id] = fields[:name]
+        begin
+          require under_score(fields[:name])
+        rescue Exception; end
+        super(fields)
+      end
+    end
+    
+    def can_authenticate?
+      self.delegate && self.delegate.singleton_methods.include?("authenticate")
+    end
+
+    def authenticate(login, password, session)
+      if self.delegate && self.delegate.authenticate(login, password, session)
+        user = User.load(login) if User.is_exist?(login)
+        if not user
+          user = User.create(:login => login)
+          users << user.id
+        end
+        return user
+      end
+    end
+    
+    # TODO: Do we need this anymore?
+    # def delete
+    #   sources.members.each do |source_name|
+    #     Source.load(source_name,{:app_id => self.name,
+    #       :user_id => '*'}).delete
+    #   end
+    #   users.members.each do |user_name|
+    #     User.load(user_name).delete
+    #   end
+    #   ReadState.delete(self.name)
+    #   super
+    # end
+    
+    def delegate
+      @delegate.nil? ? Object.const_get(camelize(self.name)) : @delegate
+    end
+    
+    def partition_sources(partition,user_id)
+      names = []
+      need_refresh = false
+      sources.members.each do |source|
+        s = Source.load(source,{:app_id => self.name,
+          :user_id => user_id})
+        if s.partition == partition
+          names << s.name
+          need_refresh = true if !need_refresh and s.check_refresh_time  
+        end
+      end
+      {:names => names,:need_refresh => need_refresh}
+    end
+    
+    def store_blob(obj,field_name,blob)
+      self.delegate.send :store_blob, obj,field_name,blob
+    end
+  end
+end

data/lib/rhosync/bulk_data/bulk_data.rb

@@ -0,0 +1,75 @@
+require 'resque'
+require 'rhosync/jobs/bulk_data_job'
+
+module Rhosync
+  class BulkData < Model
+    field :name, :string
+    field :state, :string
+    field :app_id, :string
+    field :user_id, :string
+    field :refresh_time, :integer
+    field :dbfile,:string
+    set   :sources, :string
+    validates_presence_of :app_id, :user_id, :sources
+    
+    def completed?
+      if state.to_sym == :completed and 
+        dbfile and File.exist?(dbfile)
+        return true
+      end
+      false
+    end
+    
+    def delete
+      sources.members.each do |source|
+        s = Source.load(source,{:app_id => app_id, :user_id => user_id})
+        Store.flash_data(s.docname(:md_copy)) if s
+      end
+      super
+    end
+    
+    def process_sources
+      sources.members.each do |source|
+        s = Source.load(source,{:app_id => app_id, :user_id => user_id})
+        if s
+          SourceSync.new(s).process_query(nil)
+          s.clone(:md,:md_copy) unless s.sync_type.to_sym == :bulk_sync_only
+        end
+      end
+    end
+    
+    def url
+      dbfile
+    end
+    
+    class << self
+      def create(fields={})
+        fields[:id] = fields[:name]
+        fields[:state] ||= :inprogress
+        fields[:sources] ||= []
+        super(fields)
+      end
+      
+      def enqueue(params={})
+        Resque.enqueue(BulkDataJob,params)
+      end
+      
+      def get_name(partition,client)
+        if partition == :user
+          File.join(client.app_id,client.user_id,client.user_id)
+        else
+          File.join(client.app_id,client.app_id)
+        end
+      end
+      
+      def schema_file
+        File.join(File.dirname(__FILE__),'syncdb.schema')
+      end
+      
+      def index_file
+        File.join(File.dirname(__FILE__),'syncdb.index.schema')
+      end
+    end
+  end
+end
+

data/lib/rhosync/bulk_data/syncdb.index.schema

@@ -0,0 +1,3 @@
+CREATE INDEX by_src_id on object_values (source_id);
+CREATE UNIQUE INDEX by_src_object ON object_values (object, attrib, source_id);
+CREATE INDEX by_src_value ON object_values (attrib, source_id, value);

data/lib/rhosync/bulk_data/syncdb.schema

@@ -0,0 +1,37 @@
+CREATE TABLE client_info (
+ client_id VARCHAR(255) default NULL,
+ session VARCHAR(255) default NULL,
+ token VARCHAR(255) default NULL,
+ token_sent BIGINT default 0,
+ reset BIGINT default 0,
+ port VARCHAR(10) default NULL,
+ bulksync_state BIGINT default 0,
+ last_sync_success VARCHAR(100) default NULL);
+CREATE TABLE object_values (
+ source_id BIGINT default NULL,
+ attrib varchar(255) default NULL,
+ object varchar(255) default NULL,
+ value varchar default NULL,
+ attrib_type varchar(255) default NULL);
+CREATE TABLE changed_values (
+ source_id BIGINT default NULL,
+ attrib varchar(255) default NULL,
+ object varchar(255) default NULL,
+ value varchar default NULL,
+ attrib_type varchar(255) default NULL,
+ update_type varchar(255) default NULL,
+ sent BIGINT default 0);
+CREATE TABLE sources (
+ source_id BIGINT PRIMARY KEY,
+ name VARCHAR(255) default NULL,
+ token BIGINT default NULL,
+ priority BIGINT,
+ partition VARCHAR(255),
+ sync_type VARCHAR(255),
+ last_updated BIGINT default 0,
+ last_inserted_size BIGINT default 0,
+ last_deleted_size BIGINT default 0,
+ last_sync_duration BIGINT default 0,
+ last_sync_success BIGINT default 0,
+ backend_refresh_time BIGINT default 0,
+ source_attribs varchar default NULL);

data/lib/rhosync/bulk_data.rb

@@ -0,0 +1,2 @@
+$:.unshift File.dirname(__FILE__)
+require 'bulk_data/bulk_data'

data/lib/rhosync/client.rb

@@ -0,0 +1,74 @@
+module Rhosync  
+  class InvalidSourceNameError < RuntimeError; end
+  
+  class Client < Model
+    field :device_type,:string
+    field :device_pin,:string
+    field :device_port,:string
+    
+    field :user_id,:string
+    field :app_id,:string
+    attr_accessor :source_name
+    validates_presence_of :app_id, :user_id
+    
+    include Document
+    include LockOps
+    
+    def self.create(fields,params={})
+      Rhosync.license.check_and_use_seat
+      fields[:id] = get_random_uuid
+      res = super(fields,params)
+      user = User.load(fields[:user_id])
+      user.clients << res.id
+      res
+    end
+    
+    def self.load(id,params)
+      validate_attributes(params)
+      super(id,params)
+    end
+    
+    def app
+      @app ||= App.load(app_id)
+    end
+    
+    def doc_suffix(doctype)
+      doctype = doctype.to_s
+      if doctype == '*'
+        "#{self.user_id}:#{self.id}:*"
+      elsif self.source_name 
+        "#{self.user_id}:#{self.id}:#{self.source_name}:#{doctype}"
+      else
+        raise InvalidSourceNameError.new('Invalid Source Name For Client')   
+      end          
+    end
+    
+    def delete
+      flash_data('*')
+      Rhosync.license.free_seat
+      super
+    end
+    
+    def update_clientdoc(sources)
+      sources.each do |source|
+        s = Source.load(source,{:app_id => app_id,:user_id => user_id})
+        unless s.sync_type.to_sym == :bulk_sync_only
+          self.source_name = source
+          Store.clone(s.docname(:md_copy),self.docname(:cd))
+        end
+      end
+    end
+    
+    def update_fields(params)
+      [:device_type,:device_pin,:device_port].each do |setting|
+        self.send "#{setting}=".to_sym, params[setting].to_s if params[setting]
+      end
+    end
+    
+    private
+    
+    def self.validate_attributes(params)
+      raise ArgumentError.new('Missing required attribute source_name') unless params[:source_name]
+    end
+  end
+end

data/lib/rhosync/client_sync.rb

@@ -0,0 +1,296 @@
+module Rhosync
+  class ClientSync
+    attr_accessor :source,:client,:p_size,:source_sync
+    
+    VERSION = 3
+    
+    def initialize(source,client,p_size=nil)
+      raise ArgumentError.new('Missing required attribute client') unless client
+      raise ArgumentError.new('Missing required attribute source') unless source
+      @source,@client,@p_size = source,client,p_size ? p_size.to_i : 500
+      @source_sync = SourceSync.new(@source)
+    end
+    
+    def receive_cud(cud_params={},query_params=nil)
+      _process_blobs(cud_params)
+      processed = 0
+      ['create','update','delete'].each do |op|
+        key,value = op,cud_params[op]
+        processed += _receive_cud(key,value) if value
+      end
+      @source_sync.process_cud(@client.id) if processed > 0
+    end
+    
+    def send_cud(token=nil,query_params=nil)
+      res = []
+      if not _ack_token(token)
+        res = resend_page(token)
+      else
+        @source_sync.process_query(query_params)
+        res = send_new_page
+      end
+      _format_result(res[0],res[1],res[2],res[3])
+    end
+    
+    def search(params)
+      res = []
+      return _resend_search_result(params[:search_token]) if params[:search_token] and params[:resend]
+      _ack_search(params[:search_token]) if params[:search_token]
+      res = _do_search(params[:search]) if params[:search]
+      res
+    end
+    
+    def build_page
+      res = {}
+      yield res
+      res.reject! {|key,value| value.nil? or value.empty?}
+      res.merge!(_send_errors)
+      res
+    end
+    
+    def send_new_page
+      compute_errors_page
+      token,progress_count,total_count = '',0,0
+      res = build_page do |r|
+        progress_count,total_count,r['insert'] = compute_page
+        r['delete'] = compute_deleted_page
+        r['links'] = compute_links_page
+        r['metadata'] = compute_metadata
+      end
+      if res['insert'] or res['delete'] or res['links']
+        token = compute_token(@client.docname(:page_token))
+      else
+        _delete_errors_page 
+      end    
+      @client.put_data(:cd,res['insert'],true)      
+      @client.delete_data(:cd,res['delete'])
+      [token,progress_count,total_count,res]
+    end
+    
+    # Resend token for a client, also sends exceptions
+    def resend_page(token=nil)
+      token,progress_count,total_count = '',0,0
+      res = build_page do |r|
+        r['insert'] = @client.get_data(:page)
+        r['delete'] = @client.get_data(:delete_page)
+        r['links'] = @client.get_data(:create_links_page)
+        r['metadata'] = compute_metadata
+        progress_count = @client.get_value(:cd_size).to_i
+        total_count = @client.get_value(:total_count_page).to_i
+      end
+      token = @client.get_value(:page_token)
+      [token,progress_count,total_count,res]
+    end
+    
+    # Computes the metadata sha1 and returns metadata if client's sha1 doesn't 
+    # match source's sha1
+    def compute_metadata
+      metadata_sha1,metadata = @source.lock(:metadata) do |s|
+        [s.get_value(:metadata_sha1),s.get_value(:metadata)]
+      end
+      return if @client.get_value(:metadata_sha1) == metadata_sha1
+      @client.put_value(:metadata_sha1,metadata_sha1)
+      metadata
+    end
+    
+    # Computes diffs between master doc and client doc, trims it to page size, 
+    # stores page, and returns page as hash  
+    def compute_page
+      res,diffsize,total_count = @source.lock(:md) do |s| 
+        res,diffsize = Store.get_diff_data(@client.docname(:cd),s.docname(:md),@p_size)
+        total_count = s.get_value(:md_size).to_i
+        [res,diffsize,total_count]
+      end
+      @client.put_data(:page,res)
+      progress_count = total_count - diffsize
+      @client.put_value(:cd_size,progress_count)
+      @client.put_value(:total_count_page,total_count)
+      [progress_count,total_count,res]
+    end
+    
+    # Computes search hash
+    def compute_search
+      Store.get_diff_data(@client.docname(:cd),@client.docname(:search),@p_size)
+    end
+    
+    # Computes deleted objects (down to individual attributes) 
+    # in the client document, trims it to page size, stores page, and returns page as hash      
+    def compute_deleted_page
+      res = {}
+      delete_page_doc = @client.docname(:delete_page)
+      page_size = @p_size
+      diff = @source.lock(:md) { |s| Store.get_diff_data(s.docname(:md),@client.docname(:cd))[0] }
+      diff.each do |key,value|
+        res[key] = value
+        value.each do |attrib,val|
+          Store.db.sadd(delete_page_doc,setelement(key,attrib,val))
+        end
+        page_size -= 1
+        break if page_size <= 0          
+      end
+      res
+    end
+    
+    # Computes errors for client and stores a copy as errors page
+    def compute_errors_page
+      ['create','update','delete'].each do |operation|
+        @client.lock("#{operation}_errors") do |c| 
+          c.rename("#{operation}_errors","#{operation}_errors_page")
+        end
+      end
+    end
+    
+    # Computes create links for a client and stores a copy as links page
+    def compute_links_page
+      @client.lock(:create_links) do |c| 
+        c.rename(:create_links,:create_links_page)
+        c.get_data(:create_links_page)
+      end
+    end
+        
+    class << self
+      # Resets the store for a given app,client
+      def reset(client)
+        client.flash_data('*') if client
+      end
+    
+      def search_all(client,params=nil)
+        return [] unless params[:sources]
+        res = []
+        params[:sources].each do |source|
+          s = Source.load(source,{:app_id => client.app_id,
+            :user_id => client.user_id})
+          client.source_name = source
+          cs = ClientSync.new(s,client,params[:p_size])
+          search_res = cs.search(params)
+          res << search_res if search_res
+        end
+        res
+      end
+      
+      def bulk_data(partition,client)
+        name = BulkData.get_name(partition,client)
+        data = BulkData.load(name)
+        sources = client.app.partition_sources(partition,client.user_id)
+        if (data.nil? or (data.completed? and sources[:need_refresh] == true)) and 
+          sources[:names].length > 0 
+          data.delete if data
+          data = BulkData.create(:name => name,
+            :app_id => client.app_id,
+            :user_id => client.user_id,
+            :sources => sources[:names])
+          BulkData.enqueue("data_name" => name)
+        end
+        if data and data.completed? 
+          client.update_clientdoc(sources[:names])
+          {:result => :url, :url => data.url}
+        elsif data
+          {:result => :wait}
+        else
+          {:result => :nop}
+        end
+      end
+    end
+    
+    private
+    def _resend_search_result(search_token)
+       _format_search_result
+    end
+    
+    def _ack_search(search_token)
+      token = @client.get_value(:search_token)
+      if token == search_token
+        @client.flash_data('search*')      
+      end
+    end
+    
+    def _do_search(params)
+      @source_sync.search(@client.id,params)
+      _format_search_result      
+    end
+    
+    def _format_search_result
+      error = @client.get_data(:search_errors)
+      if not error.empty?
+        [ {'version'=>VERSION},
+          {'source'=>@source.name},
+          {'search-error'=>error} ]
+      else  
+        search_token = @client.get_value(:search_token)
+        search_token ||= ''
+        res,diffsize = compute_search 
+        return [] if res.empty?
+        [ {'version'=>VERSION},
+          {'search_token' => search_token},
+          {'source'=>@source.name},
+          {'count'=>res.size},
+          {'insert'=>res} ]
+       end
+    end
+    
+    def _receive_cud(operation,params)
+      return 0 if not ['create','update','delete'].include?(operation)
+      @client.lock(operation) { |c| c.put_data(operation,params,true) }
+      return 1
+    end
+    
+    def _process_blobs(params)
+      unless params[:blob_fields].nil?
+        [:create,:update].each do |utype|
+          objects = params[utype] || {}
+          objects.each do |id,obj|
+            params[:blob_fields].each do |field|
+        		  blob = params["#{field}-#{id}"]
+        		  obj[field] = @client.app.store_blob(obj,field,blob)
+      		  end
+        	end
+      	end
+      end
+    end
+    
+    def _ack_token(token)
+      stored_token = @client.get_value(:page_token)
+      if stored_token 
+        if token and stored_token == token
+          @client.put_value(:page_token,nil)
+          @client.flash_data(:create_links_page)
+          @client.flash_data(:page)
+          @client.flash_data(:delete_page)
+          _delete_errors_page
+          return true
+        end
+      else
+        return true    
+      end    
+      false
+    end
+    
+    def _delete_errors_page
+      ['create','update','delete'].each do |operation|
+        @client.flash_data("#{operation}_errors_page")
+      end
+    end
+    
+    def _send_errors
+      res = {}
+      ['create','update','delete'].each do |operation|
+        res["#{operation}-error"] = @client.get_data("#{operation}_errors_page")
+      end
+      res["source-error"] = @source.lock(:errors) { |s| s.get_data(:errors) }
+      res.reject! {|key,value| value.nil? or value.empty?}
+      res
+    end
+    
+    def _format_result(token,progress_count,total_count,res)
+      count = 0
+      count += res['insert'].length if res['insert']
+      count += res['delete'].length if res['delete']
+      [ {'version'=>VERSION},
+        {'token'=>(token ? token : '')},
+        {'count'=>count},
+        {'progress_count'=>progress_count},
+        {'total_count'=>total_count},
+        res ]
+    end
+  end
+end