rubysync 0.1.1 → 0.2.1

data/lib/ruby_sync/connectors/connector_event_processing.rb

@@ -22,12 +22,13 @@ module RubySync
     module ConnectorEventProcessing
     
       def process(event)
+	perform_transform(:target_transform, event, name)
         case event.type
-        when :add: return perform_add(event)
-        when :delete: return perform_delete(event)
-        when :modify: return perform_modify(event)
+        when :add then return perform_add(event)
+        when :delete then return perform_delete(event)
+        when :modify then return perform_modify(event)
         else
-            raise Exception.new("#{name}: Unknown event type '#{event.type}' received")
+	  raise Exception.new("#{name}: Unknown event type '#{event.type}' received")
         end
       end
 
@@ -38,9 +39,10 @@ module RubySync
         log.info "Adding '#{event.target_path}' to '#{name}'"
         raise Exception.new("#{name}: Entry with path '#{event.target_path}' already exists, add failing.") if self[event.target_path]
         if is_vault? && event.association && path_for_association(event.association)
-          raise Exception.new("#{name}: Association (#{event.association.to_s}) already in use. Add failing.") 
+	  log.warn("#{name}: Removing obsolete association (#{event.association.to_s}) found for non-existent #{event.target_path}.")
+	  self.remove_association(event.association)
         end
-        call_if_exists(:target_transform, event)
+	#        call_if_exists(:target_transform, event)
         if add(event.target_path, event.payload)
           log.info "Add succeeded"
           update_mirror event.target_path
@@ -61,36 +63,38 @@ module RubySync
 
       def perform_delete event
         raise Exception.new("#{name}: Delete of unassociated object. No action taken.") unless event.association
-        path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
+        path = associated_path event
         log.info "Deleting '#{path}' from '#{name}'"
         delete(path) or log.warn("#{name}: Attempted to delete non-existent entry '#{path}'\nMay be an echo of a delete from this connector, ignoring.")
         delete_from_mirror path
         return nil # don't want to create any new associations
       end
       
-      def delete_from_mirror path
-        DBM.open(self.mirror_dbm_filename) do |dbm|
-          dbm.delete(path)
-        end
-      end
-
       def perform_modify event
-        path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
+        path = associated_path event
         raise Exception.new("#{name}: Attempted to modify non-existent entry '#{path}'") unless self[path]
-        call_if_exists(:target_transform, event)
+        #call_if_exists(:target_transform, event)
         modify path, event.payload
         update_mirror path
         return (is_vault?)? nil : own_association_key_for(event.target_path)
       end
       
       def update_mirror path
-        if entry = self[path]
-          DBM.open(self.mirror_dbm_filename) do |dbm|
-            dbm[path.to_s] = digest(entry)
-           end
-        end
       end
       
+      def delete_from_mirror path
+      end
+      
+      def clean
+        remove_associations if respond_to? :remove_associations
+        remove_mirror if respond_to? :remove_mirror
+      end
+      
+      def associated_path event
+	(is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
+      end
+      
+      
     end
   end
 end

data/lib/ruby_sync/connectors/csv_file_connector.rb

@@ -6,9 +6,8 @@
 require "csv"
 require "ruby_sync/connectors/file_connector"
 
-module RubySync
+module RubySync::Connectors
   
-  module Connectors
     
     # Reads files containing Comma Separated Values from the in_path directory and
     # treats each line as an incoming event.
@@ -27,7 +26,7 @@ module RubySync
       
       
       # Called for each filename matching in_glob in in_path
-      # Yields a modify event for each row found in the file.
+      # Yields an add event for each row found in the file.
       def each_file_change(filename)
         header = header_line
         CSV.open(filename, 'r') do |row|
@@ -46,7 +45,7 @@ module RubySync
             row[i] and data[field_name] = row[i].data
           end
           association_key = source_path = path_for(data)
-          yield RubySync::Event.modify(self, source_path, association_key, create_operations_for(data))
+          yield RubySync::Event.add(self, source_path, association_key, create_operations_for(data))
         end
       end
 
@@ -102,4 +101,3 @@ END
       
     end
   end
-end

data/lib/ruby_sync/connectors/dbm_association_tracking.rb

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+require 'yaml'
+require 'yaml/dbm'
+ 
+
+module RubySync
+  module Connectors
+    module DbmAssociationTracking
+        
+            # Store association for the given path
+      def associate association, path
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s] || {}
+          assocs[association.context.to_s] = association.key.to_s
+          dbm[path.to_s] = assocs
+        end
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          dbm[association.to_s] = path
+        end
+      end
+      
+      def path_for_association association
+        is_vault? or return path_for_own_association_key(association.key)
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          dbm[association.to_s]
+        end
+      end
+      
+      def associations_for path
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs =  dbm[path.to_s]
+          assocs.values
+        end
+      end
+
+
+      def remove_association association
+        path = nil
+        DBM.open(association_to_path_dbm_filename) do |dbm|
+          return unless path =dbm.delete(association.to_s)
+        end
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s]
+          assocs.delete(association.context) and dbm[path.to_s] = assocs
+        end
+      end
+
+      def association_key_for context, path
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s] || {}
+          assocs[context.to_s]
+        end
+      end
+
+      def remove_associations
+        File.delete_if_exists(["#{association_to_path_dbm_filename}.db","#{path_to_association_dbm_filename}.db"])
+      end
+
+      
+            # Stores association keys indexed by path:association_context
+        def path_to_association_dbm_filename
+          dbm_path + "_path_to_assoc"
+        end
+
+      # Stores paths indexed by association_context:association_key
+        def association_to_path_dbm_filename
+          dbm_path + "_assoc_to_path"
+        end
+
+
+      
+    end
+  end
+end

data/lib/ruby_sync/connectors/dbm_change_tracking.rb

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+require 'yaml'
+require 'yaml/dbm'
+require 'digest/md5'
+
+# When included by a connector, tracks changes to the connector using
+# a dbm database of the form path:digest. Digest is a MD5 hash of the
+# record so we can tell if the record has changed. We can't, however,
+# tell how the record has changed so change events generated will be
+# for the entire record.
+module RubySync::Connectors::DbmChangeTracking
+
+  option  :dbm_path
+
+  # set a default dbm path
+  def dbm_path()
+    p = "#{base_path}/db"
+    ::FileUtils.mkdir_p p
+    ::File.join(p,name)
+  end
+
+  # Stores a hash for each entry so we can tell when
+  # entries are added, deleted or modified
+  def mirror_dbm_filename
+    dbm_path + "_mirror"
+  end
+      
+  # Subclasses MAY override this to interface with the external system
+  # and generate an event for every change that affects items within
+  # the scope of this connector.
+  #
+  # The default behaviour is to compare a hash of each entry in the
+  # database with a stored hash of its previous value and generate
+  # add, modify and delete events appropriately. This is normally a very
+  # inefficient way to operate so overriding this method is highly
+  # recommended if you can detect changes in a more efficient manner.
+  #
+  # This method will be called repeatedly until the connector is
+  # stopped.
+  def each_change
+    DBM.open(self.mirror_dbm_filename) do |dbm|
+      # scan existing entries to see if any new or modified
+      each_entry do |path, entry|
+	digest = digest(entry)
+	unless stored_digest = dbm[path.to_s] and digest == stored_digest
+	  operations = create_operations_for(entry)
+	  yield RubySync::Event.add(self, path, nil, operations) 
+	  dbm[path.to_s] = digest
+	end
+      end
+          
+      # scan dbm to find deleted
+      dbm.each do |key, stored_hash|
+	unless self[key]
+	  yield RubySync::Event.delete(self, key)
+	  dbm.delete key
+	  if is_vault? and @pipeline
+	    association = association_for @pipeline.association_context, key
+	    remove_association association
+	  end
+	end
+      end
+    end        
+  end
+      
+  def digest(o)
+    Digest::MD5.hexdigest(o.to_yaml)
+  end
+      
+
+
+  def remove_mirror
+    File.delete_if_exists(["#{mirror_dbm_filename}.db"])
+  end
+      
+  def delete_from_mirror path
+    DBM.open(self.mirror_dbm_filename) do |dbm|
+      dbm.delete(path)
+    end
+  end
+
+  def update_mirror path
+    if entry = self[path]
+      DBM.open(self.mirror_dbm_filename) do |dbm|
+	dbm[path.to_s] = digest(entry)
+      end
+    end
+  end
+      
+  def mirror_dbm_filename
+    dbm_path + "_mirror"
+  end
+
+
+end 

data/lib/ruby_sync/connectors/ldap_changelog_connector.rb

@@ -27,100 +27,101 @@ require 'net/ldap'
 class LdapChangelogConnector < RubySync::Connectors::LdapConnector
 
   option             :changelog_dn
-      changelog_dn          "cn=changelog"
+  changelog_dn          "cn=changelog"
       
-    def initialize options={}
-      super options 
-      @last_change_number = 1
-      # TODO: Persist the current CSN, for now we'll just skip to the end of the changelog
-      skip_existing_changelog_entries
-    end
-    # Look for changelog entries. This is not supported by all LDAP servers
-    # Changelog entries have these attributes
-    # targetdn
-    # changenumber
-    # objectclass
-    # changes
-    # changetime
-    # changetype
-    # dn
-    #
-    # TODO: Detect presence/location of changelog from root DSE
-    def each_change
-      with_ldap do |ldap|
-        log.debug "@last_change_number = #{@last_change_number}"
-        filter = "(changenumber>=#{@last_change_number})"
-        first = true
-        @full_refresh_required = false
-        ldap.search :base => changelog_dn, :filter =>filter do |change|
-          change_number = change.changenumber[0].to_i
-          if first
-            first = false
-            # TODO: Persist the change_number so that we don't do a full resync everytime rubysync starts
-            if change_number != @last_change_number
-              log.warn "Earliest change number (#{change_number}) differs from that recorded (#{@last_change_number})."
-              log.warn "A full refresh is required."
-              @full_refresh_required = true
-              break
-            end
-          else
-            @last_change_number = change_number if change_number > @last_change_number
-            # todo: A proper DN object would be nice instead of string manipulation
-            target_dn = change.targetdn[0].gsub(/\s*,\s*/,',')
-            if target_dn =~ /#{search_base}$/oi
-              change_type = change.changetype[0]
-              event = event_for_changelog_entry(change)
-              yield event
-            end
-          end
-        end
+  def initialize options={}
+    super options 
+    @last_change_number = 1
+    # TODO: Persist the current CSN, for now we'll just skip to the end of the changelog
+    skip_existing_changelog_entries
+  end
+  # Look for changelog entries. This is not supported by all LDAP servers
+  # Changelog entries have these attributes
+  # targetdn
+  # changenumber
+  # objectclass
+  # changes
+  # changetime
+  # changetype
+  # dn
+  #
+  # TODO: Detect presence/location of changelog from root DSE
+  def each_change
+    with_ldap do |ldap|
+      log.debug "@last_change_number = #{@last_change_number}"
+      filter = "(changenumber>=#{@last_change_number})"
+      first = true
+      @full_refresh_required = false
+      ldap.search :base => changelog_dn, :filter =>filter do |change|
+	change_number = change.changenumber[0].to_i
+	if first
+	  first = false
+	  # TODO: Persist the change_number so that we don't do a full resync everytime rubysync starts
+	  if change_number != @last_change_number
+	    log.warn "Earliest change number (#{change_number}) differs from that recorded (#{@last_change_number})."
+	    log.warn "A full refresh is required."
+	    @full_refresh_required = true
+	    break
+	  end
+	else
+	  @last_change_number = change_number if change_number > @last_change_number
+	  # TODO: A proper DN object would be nice instead of string manipulation
+	  target_dn = change.targetdn[0].gsub(/\s*,\s*/,',')
+	  if target_dn =~ /#{search_base}$/oi
+	    change_type = change.changetype[0]
+	    event = event_for_changelog_entry(change)
+	    yield event
+	  end
+	end
       end
     end
+    each_entry if @full_refresh_required
+  end
     
     
-    def skip_existing_changelog_entries
-      with_ldap do |ldap|
-        filter = "(changenumber>=#{@last_change_number})"
-        @full_refresh_required = false
-        ldap.search :base => changelog_dn, :filter =>filter do |change|
-          change_number = change.changenumber[0].to_i
-          @last_change_number = change_number if change_number > @last_change_number
-        end
+  def skip_existing_changelog_entries
+    with_ldap do |ldap|
+      filter = "(changenumber>=#{@last_change_number})"
+      @full_refresh_required = false
+      ldap.search :base => changelog_dn, :filter =>filter do |change|
+	change_number = change.changenumber[0].to_i
+	@last_change_number = change_number if change_number > @last_change_number
       end
     end
+  end
 
     
-    # Called by unit tests to inject data
-    def test_add id, details
-      details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson', 'organizationalPerson', 'person', 'top', 'RubySyncSynchable'])
-      add id, details
-    end
+  # Called by unit tests to inject data
+  def test_add id, details
+    details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson', 'organizationalPerson', 'person', 'top', 'RubySyncSynchable'])
+    add id, details
+  end
     
     
   private
   
   
-    def event_for_changelog_entry cle
-      payload = nil
-      dn = cle.targetdn[0]
-      changetype = cle.changetype[0]
-      if cle.attribute_names.include? :changes
-        payload = []
-        cr = Net::LDIF.parse("dn: #{dn}\nchangetype: #{changetype}\n#{cle.changes[0]}")[0]
-        if  changetype.to_sym == :add
-          # cr.data will be a hash of arrays or strings (attr-name=>[value1, value2, ...])
-          cr.data.each do |name, values|
-            payload << RubySync::Operation.add(name, values)
-          end
-        else
-          # cr.data will be an array of arrays of form [:action, :subject, [values]]
-          cr.data.each do |record|
-            payload << RubySync::Operation.new(record[0], record[1], record[2])
-          end
-        end
+  def event_for_changelog_entry cle
+    payload = nil
+    dn = cle.targetdn[0]
+    changetype = cle.changetype[0]
+    if cle.attribute_names.include? :changes
+      payload = []
+      cr = Net::LDIF.parse("dn: #{dn}\nchangetype: #{changetype}\n#{cle.changes[0]}")[0]
+      if  changetype.to_sym == :add
+	# cr.data will be a hash of arrays or strings (attr-name=>[value1, value2, ...])
+	cr.data.each do |name, values|
+	  payload << RubySync::Operation.add(name, values)
+	end
+      else
+	# cr.data will be an array of arrays of form [:action, :subject, [values]]
+	cr.data.each do |record|
+	  payload << RubySync::Operation.new(record[0], record[1], record[2])
+	end
       end
-      RubySync::Event.new(changetype, self, dn, nil, payload)
     end
+    RubySync::Event.new(changetype, self, dn, nil, payload)
+  end
   
 end