rubysync 0.0.2 → 0.0.3

data/examples/ar_webapp/public/.htaccess

@@ -0,0 +1,40 @@
+# General Apache options
+AddHandler fastcgi-script .fcgi
+AddHandler cgi-script .cgi
+Options +FollowSymLinks +ExecCGI
+
+# If you don't want Rails to look in certain directories,
+# use the following rewrite rules so that Apache won't rewrite certain requests
+# 
+# Example:
+#   RewriteCond %{REQUEST_URI} ^/notrails.*
+#   RewriteRule .* - [L]
+
+# Redirect all requests not available on the filesystem to Rails
+# By default the cgi dispatcher is used which is very slow
+# 
+# For better performance replace the dispatcher with the fastcgi one
+#
+# Example:
+#   RewriteRule ^(.*)$ dispatch.fcgi [QSA,L]
+RewriteEngine On
+
+# If your Rails application is accessed via an Alias directive,
+# then you MUST also set the RewriteBase in this htaccess file.
+#
+# Example:
+#   Alias /myrailsapp /path/to/myrailsapp/public
+#   RewriteBase /myrailsapp
+
+RewriteRule ^$ index.html [QSA]
+RewriteRule ^([^.]+)$ $1.html [QSA]
+RewriteCond %{REQUEST_FILENAME} !-f
+RewriteRule ^(.*)$ dispatch.cgi [QSA,L]
+
+# In case Rails experiences terminal errors
+# Instead of displaying this message you can supply a file here which will be rendered instead
+# 
+# Example:
+#   ErrorDocument 500 /500.html
+
+ErrorDocument 500 "<h2>Application error</h2>Rails application failed to start properly"

data/gemspec

@@ -0,0 +1,48 @@
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+require 'rubygems'
+
+SPEC = Gem::Specification.new do |s|
+  s.name = "rubysync"
+  s.version = "0.0.3"
+  s.rubyforge_project = 'rubysync'
+  s.author = "Ritchie Young"
+  s.email = "ritchiey@gmail.com"
+  s.homepage = "http://rubysync.org"
+  #s.platform = "Gem::Platform::RUBY"
+  s.summary = "Event driven identity synchronization engine"
+  candidates = Dir.glob "{bin,docs,lib,test,examples}/**/*"
+  s.files = candidates.delete_if do |item|
+    item.include?("rubysync.tmproj") ||
+    item.include?(".svn") ||
+    item.include?(".project") ||
+    item.include?('.DS_Store') ||
+    item.include?('tmp') ||
+    item.include?('log')
+  end
+  s.bindir = 'bin'
+  s.require_path = 'lib'
+  s.executables = ['rubysync']
+  s.default_executable = 'rubysync'
+  s.autorequire = 'ruby_sync.rb'
+  s.test_file = 'test/ts_rubysync.rb'
+  s.has_rdoc = true
+  #s.extra_rdoc_files = ["README"]
+  s.add_dependency "ruby-net-ldap", ">=0.0.4"
+  s.add_dependency "activesupport", ">=1.4.0"
+  s.add_dependency "activerecord", ">=1.15.3"
+  s.add_dependency "simpleconsole", ">=0.1.1"
+  s.add_dependency "xmlsimple", ">=1.0.11"
+end

data/lib/ruby_sync/connectors/active_record_association_handler.rb

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby -w
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+lib_path = File.dirname(__FILE__) + '/../lib'
+$:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
+
+require 'ruby_sync'
+
+module RubySync::Connectors::ActiveRecordAssociationHandler
+  
+  def associate association, path
+    log.debug "Associating '#{association}' with '#{path}'"
+    ruby_sync_association.create :synchronizable_id=>path, :synchronizable_type=>ar_class.name,
+                                 :context=>association.context, :key=>association.key
+  end
+  
+  def find_associated association
+    ruby_sync_association.find_by_context_and_key association.context, association.key
+  end
+  
+  def path_for_association association
+    assoc = ruby_sync_association.find_by_context_and_key association.context, association.key
+    (assoc)? assoc.synchronizable_id : nil
+  end
+  
+  def association_key_for context, path
+    record = ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type_and_context path, model.to_s, context
+    record and record.key
+  end
+  
+  def associations_for(path)
+    ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type(path, model.to_s)
+  rescue ActiveRecord::RecordNotFound
+    return nil
+  end
+  
+  def remove_association association
+     ruby_sync_association.find_by_context_and_key(association.context, association.key).destroy
+  rescue ActiveRecord::RecordNotFound
+     return nil
+  end
+  
+private
+
+  def ruby_sync_association
+    unless @ruby_sync_association
+      @ruby_sync_association = ::RubySyncAssociation
+      ::RubySyncAssociation.establish_connection(db_config)
+    end
+    @ruby_sync_association
+  end
+
+end

data/lib/ruby_sync/connectors/active_record_connector.rb

@@ -66,6 +66,7 @@ module RubySync::Connectors
       self.class.ar_class model.to_s.camelize.constantize
     end
 
+
       
       def self.fields
         c = self.new
@@ -81,22 +82,14 @@ module RubySync::Connectors
 END
       end
       
-      
-    # Process each RubySyncEvent and then delete it from the db.
-    def each_change
-      ::RubySyncEvent.find(:all).each do |rse|
-        event = RubySync::Event.new(rse.event_type, self, rse.trackable_id, nil, to_payload(rse))
-        yield event
-        ::RubySyncEvent.delete rse
+    
+    def each_entry
+      ar_class.find :all do |record|
+        yield entry_from_active_record(record)
       end
     end
+      
 
-    # Create a hash suitable to use as rubysync event payload
-    def to_payload ar_event
-      ar_event.operations.map do |op|
-        RubySync::Operation.new(op.operation.to_sym, op.field_name, op.values.map {|v| v.value})
-      end
-    end
 
     # Override default perform_add because ActiveRecord is different in that
     # the target path is ignored when adding a record. ActiveRecord determines
@@ -107,6 +100,7 @@ END
         populate(record, perform_operations(event.payload))
         log.info(record.inspect)
         record.save!
+        update_mirror record.id
         if is_vault?
           associate event.association, record.id
         end
@@ -114,7 +108,6 @@ END
       end
     rescue => ex
       log.warn ex
-      #puts ex.backtrace.join("\n")
       return nil
     end
 
@@ -129,71 +122,30 @@ END
     def delete(path)
       ar_class.destroy path
     end
-  
-    # Implement vault functionality
-
-    def associate association, path
-      log.debug "Associating '#{association}' with '#{path}'"
-      ruby_sync_association.create :synchronizable_id=>path, :synchronizable_type=>ar_class.name,
-                                   :context=>association.context, :key=>association.key
-    end
-
-    def find_associated association
-      ruby_sync_association.find_by_context_and_key association.context, association.key
-    end
-
-    def path_for_association association
-      assoc = ruby_sync_association.find_by_context_and_key association.context, association.key
-      (assoc)? assoc.synchronizable_id : nil
-    end
-
-    def association_key_for context, path
-      record = ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type_and_context path, model.to_s, context
-      record and record.key
-    end
-    
-    def associations_for(path)
-      ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type(path, model.to_s)
-    rescue ActiveRecord::RecordNotFound
-      return nil
-    end
-    
-    def remove_association association
-       ruby_sync_association.find_by_context_and_key(association.context, association.key).destroy
-    rescue ActiveRecord::RecordNotFound
-       return nil
-    end
-
 
     def [](path)
-      ar_class.find(path)
+      entry_from_active_record(ar_class.find(path))
     rescue ActiveRecord::RecordNotFound
       return nil
     end
 
 private
 
-    def ruby_sync_association
-      unless @ruby_sync_association
-        @ruby_sync_association = ::RubySyncAssociation
-        ::RubySyncAssociation.establish_connection(db_config)
-      end
-      @ruby_sync_association
-    end
-
     def populate record, content
       ar_class.content_columns.each do |c|
         record[c.name] = content[c.name][0] if content[c.name]
       end
     end
 
-    def create_operations_for_active_record record
-      operations = record.class.content_columns.map do |col|
+    def entry_from_active_record record
+      entry = {}
+      record.class.content_columns.each do |col|
         key = col.name
         value = record.send key
-        RubySync::Operation.new(:add, key.to_s, value) if key and value
+        entry[key.to_s] = value if key and value
       end
-      operations.compact
+      entry
     end
+    
   end
 end

data/lib/ruby_sync/connectors/active_record_event_handler.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby -w
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+lib_path = File.dirname(__FILE__) + '/../lib'
+$:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
+
+require 'ruby_sync'
+
+module RubySync::Connectors::ActiveRecordEventHandler
+
+  # Disable mirror updates for incoming events
+  def update_mirror(path); end
+  def delete_from_mirror(path); end
+
+  # Process each RubySyncEvent and then delete it from the db.
+  def each_change
+    ::RubySyncEvent.find(:all).each do |rse|
+      event = RubySync::Event.new(rse.event_type, self, rse.trackable_id, nil, to_payload(rse))
+      yield event
+      ::RubySyncEvent.delete rse
+    end
+  end
+
+  # Create a hash suitable to use as rubysync event payload
+  def to_payload ar_event
+    ar_event.operations.map do |op|
+      RubySync::Operation.new(op.operation.to_sym, op.field_name, op.values.map {|v| v.value})
+    end
+  end
+
+
+  
+
+end

data/lib/ruby_sync/connectors/base_connector.rb

@@ -14,6 +14,8 @@
 # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
 
 require 'ruby_sync/connectors/connector_event_processing'
+require 'dbm'
+require 'digest/md5'
 
 module RubySync::Connectors
     class BaseConnector
@@ -22,8 +24,30 @@ module RubySync::Connectors
       include ConnectorEventProcessing
       
       attr_accessor :once_only, :name, :is_vault
+      option  :dbm_path
+      
+      # set a default dbm path
+      def dbm_path() "#{base_path}/db/#{name}"; end
+
+      # Stores association keys indexed by path:association_context
+      def path_to_association_dbm_filename
+        dbm_path + "_path_to_assoc"
+      end
+      
+      # Stores paths indexed by association_context:association_key
+      def association_to_path_dbm_filename
+        dbm_path + "_assoc_to_path"
+      end
+      
+      # Stores a hash for each entry so we can tell when
+      # entries are added, deleted or modified
+      def mirror_dbm_filename
+        dbm_path + "_mirror"
+      end
       
       def initialize options={}
+        base_path # call this once to get the working directory before anything else
+                  # in the connector changes the cwd
         options = self.class.default_options.merge(options)
         once_only = false
         self.name = options[:name]
@@ -48,20 +72,49 @@ module RubySync::Connectors
       
       # Override this to perform actions that must be performed the
       # when the connector starts running. (Eg, opening network connections)
-      def started; end
+      def started
+      end
       
       # Subclasses must override this to
-      # interface with the external system and generate events for every
-      # entry in the scope.
-      # These events are yielded to the passed in block to process.
+      # interface with the external system and generate entries for every
+      # entry in the scope passing the entry path (id) and its data (as a hash of arrays).
       # This method will be called repeatedly until the connector is
       # stopped.
-      def each_entry; end
+      def each_entry
+        raise "Not implemented"
+      end
 
       # Subclasses must override this to interface with the external system
       # and generate an event for every change that affects items within
       # the scope of this connector.
-      def each_change; end
+      # todo: Make the default behaviour to build a database of the key of
+      # each entry with a hash of the contents of the entry. Then to compare
+      # that against each entry to see if it has changed.
+      def each_change
+        DBM.open(self.mirror_dbm_filename) do |dbm|
+          # scan existing entries to see if any new or modified
+          each_entry do |path, entry|
+            digest = digest(entry)
+            unless stored_digest = dbm[path.to_s] and digest == stored_digest
+              operations = create_operations_for(entry)
+              yield RubySync::Event.add(self, path, nil, operations) 
+              dbm[path.to_s] = digest
+            end
+          end
+          
+          # scan dbm to find deleted
+          dbm.each do |key, stored_hash|
+            unless self[key]
+              yield RubySync::Event.delete(self, key)
+              dbm.delete key
+            end
+          end
+        end        
+      end
+      
+      def digest(o)
+        Digest::MD5.hexdigest(Marshal.dump(o))
+      end
       
       # Override this to perform actions that must be performed when
       # the connector exits (eg closing network conections).
@@ -71,12 +124,16 @@ module RubySync::Connectors
       # Call each_change repeatedly (or once if in once_only mode)
       # to generate events.
       # Should generally only be called by the pipeline to which it is attached.
-      def start
+      def start &blk
         log.info "#{name}: Started"
         @running = true
         started()
         while @running
           each_change do |event|
+            if event.type == :force_resync
+              each_entry &blk
+              next
+            end
             if is_delete_echo?(event) || is_echo?(event)
               log.debug "Ignoring echoed event"
             else
@@ -106,12 +163,6 @@ module RubySync::Connectors
       def is_vault?
         @is_vault
       end
-      
-      # Returns the correct id for the given association 
-      def path_for_association(association)
-        (is_vault?)?
-          path_for_foreign_key(association) : path_for_own_association_key(association.key)
-      end
 
 
       # Returns the association key for the given path. Called if this connector is
@@ -161,37 +212,77 @@ module RubySync::Connectors
         defined? associations_for
       end
 
-      # Implement these if you want your connector to be able to act as a vault
 
-      # def associate association, path
-      # end
-      # 
-      # def path_for_association association
-      # end
-      # 
-      #
-      # def associations_for path
-      # end
-      #
-      # def remove_association association
-      # end
+      # Store association for the given path
+      def associate association, path
+        DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs_string = dbm[path.to_s]
+          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+          assocs[association.context] = association.key
+          dbm[path.to_s] = Marshal.dump(assocs)
+        end
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          dbm[association.to_s] = path
+        end
+      end
+      
+      def path_for_association association
+        is_vault? or return path_for_own_association_key(association.key)
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          dbm[association.to_s]
+        end
+      end
+      
+      # Default implementation does nothing
+      def associations_for path
+        DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs_string = dbm[path.to_s]
+          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+          assocs.values
+        end
+      end
 
+      # Default implementation does nothing
+      def remove_association association
+        path = nil
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          return unless path =dbm.delete(association.to_s)
+        end
+        DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs_string = dbm[path]
+          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+          assocs.delete(association.context) and dbm[path.to_s] = Marshal.dump(assocs)
+        end
+      end
+
+      # Could be more efficient for the default case where the
+      # associations are actually stored as a serialized hash but
+      # then it wouldn't be as generic and other implementations would
+      # have to reimplement it.
+      # def association_key_for context, path
+      #   raise "#{name} is not a vault." unless is_vault?
+      #   associations_for(path).each do |assoc|
+      #     (c, key) = assoc.split(RubySync::Association.delimiter, 2)
+      #     return key if c == context 
+      #   end
+      #   return nil
+      # end
 
       def association_key_for context, path
-        raise "#{name} is not a vault." unless is_vault?
-        associations_for(path).each do |assoc|
-          (c, key) = assoc.split(RubySync::Association.delimiter, 2)
-          return key if c == context 
+        DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs_string = dbm[path.to_s]
+          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+          assocs[context.to_s]
         end
-        return nil
       end
+
       
       # Return the association object given the association context and path.
       # This should only be called on the vault.
       def association_for(context, path)
         raise "#{name} is not a vault." unless is_vault?
         key = association_key_for context, path
-        RubySync::Association.new(context, key)
+        key and RubySync::Association.new(context, key)
       end
 
       # Should only be called on the vault. Returns the entry associated with
@@ -200,14 +291,27 @@ module RubySync::Connectors
       # association.
       def find_associated association
         path = path_for_association association
-        self[path]
+        path and self[path]
       end
       
       # The context to be used to for all associations created where this
       # connector is the client.
       def association_context
         self.name
-      end      
+      end
+      
+      def remove_mirror
+        File.delete_if_exists(["#{mirror_dbm_filename}.db"])
+      end
+      
+      def remove_associations
+        File.delete_if_exists(["#{association_to_path_dbm_filename}.db","#{path_to_association_dbm_filename}.db"])
+      end
+
+      def clean
+        remove_associations
+        remove_mirror
+      end
       
       # Attempts to delete non-existent items may occur due to echoing. Many systems won't be able to record
       # the fact that an entry has been deleted by rubysync because after the delete, there is no entry left to
@@ -266,9 +370,8 @@ module RubySync::Connectors
           when :add
             if record[op.subject]
               existing = record[op.subject].as_array
-              unless (existing & op.values).empty?
+              (existing & op.values).empty? or
                 raise Exception.new("Attempt to add duplicate elements to #{name}")
-              end
               record[op.subject] =  existing + op.values
             else
               record[op.subject] = op.values

data/lib/ruby_sync/connectors/connector_event_processing.rb

@@ -43,6 +43,7 @@ module RubySync
         call_if_exists(:target_transform, event)
         if add(event.target_path, event.payload)
           log.info "Add succeeded"
+          update_mirror event.target_path
           if is_vault?
             if event.association
               associate(event.association, event.target_path)
@@ -63,16 +64,33 @@ module RubySync
         path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
         log.info "Deleting '#{path}' from '#{name}'"
         delete(path) or log.warn("#{name}: Attempted to delete non-existent entry '#{path}'\nMay be an echo of a delete from this connector, ignoring.")
+        delete_from_mirror path
         return nil # don't want to create any new associations
       end
+      
+      def delete_from_mirror path
+        DBM.open(self.mirror_dbm_filename) do |dbm|
+          dbm.delete(path)
+        end
+      end
 
       def perform_modify event
         path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
         raise Exception.new("#{name}: Attempted to modify non-existent entry '#{path}'") unless self[path]
         call_if_exists(:target_transform, event)
         modify path, event.payload
+        update_mirror path
         return (is_vault?)? nil : own_association_key_for(event.target_path)
       end
+      
+      def update_mirror path
+        if entry = self[path]
+          DBM.open(self.mirror_dbm_filename) do |dbm|
+            dbm[path.to_s] = digest(entry)
+          end
+        end
+      end
+      
     end
   end
 end

data/lib/ruby_sync/connectors/csv_file_connector.rb

@@ -84,6 +84,9 @@ END
         nil
       end
       
+      def delete(path)
+        log.warn "Delete on CSV driver ignored for #{path}"
+      end
       
     end
   end

data/lib/ruby_sync/connectors/file_connector.rb

@@ -31,6 +31,7 @@ module RubySync
           log.error "in_glob not set on file connector. No files will be processed"
           return
         end
+        base_path # Call it before changing directory
         log.info "#{name}: Scanning #{in_path} for #{in_glob} files..."
         Dir.chdir(in_path) do |path|
           Dir.glob(in_glob) do |filename|