rubysync 0.1.1 → 0.2.1

data/lib/ruby_sync/connectors/ldap_connector.rb

@@ -3,19 +3,26 @@
 #  Copyright (c) 2007 Ritchie Young. All rights reserved.
 #
 # This file is part of RubySync.
-# 
-# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
-# 
-# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
-# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-# 
-# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
-# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+#
+# RubySync is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of
+# the License, or (at your option) any later version.
+#
+# RubySync is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See
+# the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA
 
 
 lib_path = File.dirname(__FILE__) + '/..'
-$:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
+$:.unshift lib_path unless $:.include?(lib_path) ||
+  $:.include?(File.expand_path(lib_path))
 
 require 'ruby_sync'
 require 'net/ldif'
@@ -26,27 +33,29 @@ require 'net/ldap'
 
 class Net::LDAP::Entry
   def to_hash
-    return @myhash.dup
+    @myhash.dup
   end
 end
 
 module RubySync::Connectors
   class LdapConnector < RubySync::Connectors::BaseConnector
-    
+
     option  :host,
-    :port,
-    :bind_method,
-    :username,
-    :password,
-    :search_filter,
-    :search_base,
-    :association_attribute # name of the attribute in which to store the association key(s)
-            
+      :port,
+      :bind_method,
+      :encryption,
+      :username,
+      :password,
+      :search_filter,
+      :search_base,
+      :association_attribute # name of the attribute in which to store the association key(s)
+
     association_attribute 'RubySyncAssociation'
     bind_method           :simple
     host                  'localhost'
     port                  389
     search_filter         "cn=*"
+    encryption		 nil
 
     def initialize options={}
       super options
@@ -55,17 +64,19 @@ module RubySync::Connectors
 
     def started
       #TODO: If vault, check the schema to make sure that the association_attribute is there
+      @connections = []
+      @connection_index = 0
     end
-    
+
 
     def each_entry
       Net::LDAP.open(:host=>host, :port=>port, :auth=>auth) do |ldap|
-        ldap.search :base => search_base, :filter => search_filter, :return_result => false do |ldap_entry|
-          yield ldap_entry.dn, to_entry(ldap_entry)
-        end
+	ldap.search :base => search_base, :filter => search_filter, :return_result => false do |ldap_entry|
+	  yield ldap_entry.dn, to_entry(ldap_entry)
+	end
       end
     end
-    
+
     # Runs the query specified by the config, gets the objectclass of the first
     # returned object and returns a list of its allowed attributes
     def self.fields
@@ -73,19 +84,30 @@ module RubySync::Connectors
       log.warn "Returning a likely sample set."
       %w{ cn givenName sn }
     end
-    
+
 
 
     def self.sample_config
       return <<END
-      
-   host           'localhost'
-   port            389
-   username       'cn=Manager,dc=my-domain,dc=com'
-   password       'secret'
-   search_filter  "cn=*"
-   search_base    "ou=users,o=my-organization,dc=my-domain,dc=com"
-   #:bind_method  :simple
+
+  # Using :memory is ok for testing.
+  # For production, you will need to change to a persistent form of tracking
+  # such as :dbm or :ldap. 
+  track_changes_with :memory
+  track_associations_with :memory
+
+  host           'localhost'
+  port            389
+  username       'cn=Manager,dc=my-domain,dc=com'
+  password       'secret'
+  search_filter  "cn=*"
+  search_base    "ou=users,o=my-organization,dc=my-domain,dc=com"
+  #bind_method  :simple
+  
+  #Uncomment the following for LDAPS. If you do, make sure that
+  #you're using the LDAPS port (probably 636) and be aware that
+  #the server's certificate WON'T be checked for validity.
+  #encryption	 :simple_tls
 END
     end
 
@@ -94,11 +116,12 @@ END
     def add(path, operations)
       result = nil
       with_ldap do |ldap|
-        attributes = perform_operations(operations)
-        result = ldap.add :dn=>path, :attributes=>attributes
+	attributes = perform_operations(operations)
+	attributes['objectclass'] || log.warn("Add without objectclass attribute is unlikely to work.")
+	result = ldap.add :dn=>path, :attributes=>attributes
       end
       log.debug("ldap.add returned '#{result}'")
-      return true
+      return result
     rescue Exception
       log.warn "Exception occurred while adding LDAP record"
       log.debug $!
@@ -116,23 +139,23 @@ END
 
     def [](path)
       with_ldap do |ldap|
-        result = ldap.search :base=>path, :scope=>Net::LDAP::SearchScope_BaseObject, :filter=>'objectclass=*'
-        return nil if !result or result.size == 0
-        # todo: See if this can be shortened
-        answer = {}
-        result[0].attribute_names.each do |name|
-          answer[name.to_s] = result[0][name]
-        end
-        answer
+	result = ldap.search :base=>path, :scope=>Net::LDAP::SearchScope_BaseObject, :filter=>'objectclass=*'
+	return nil if !result or result.size == 0
+	answer = {}
+	result[0].attribute_names.each do |name|
+	  name = name.to_s.downcase
+	  answer[name] = result[0][name] unless name == 'dn'
+	end
+	answer
       end
     end
-    
+
     # Called by unit tests to inject data
     def test_add id, details
       details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson'])
       add id, details
     end
-    
+
     def target_transform event
       #event.add_default 'objectclass', 'inetOrgUser'
       #is_vault? and event.add_value 'objectclass', RUBYSYNC_ASSOCIATION_CLASS
@@ -145,7 +168,7 @@ END
     def to_entry ldap_entry
       entry = {}
       ldap_entry.each do |name, values|
-        entry[name.to_s] = values.map {|v| String.new(v)}
+	entry[name.to_s] = values.map {|v| String.new(v)}
       end
       entry
     end
@@ -153,31 +176,35 @@ END
     def operations_for_entry entry
       ops = []
       entry.each do |name, values|
-        ops << RubySync::Operation.add(name, values)
+	ops << RubySync::Operation.add(name, values)
       end
       ops
     end
 
-    
-
-
 
     def with_ldap
       result = nil
-      Net::LDAP.open(:host=>host, :port=>port, :auth=>auth) do |ldap|
-        result = yield ldap
+      connection_options = {:host=>host, :port=>port, :auth=>auth}
+      connection_options[:encryption] = encryption if encryption
+      @connections[@connection_index] = Net::LDAP.new(connection_options) unless @connections[@connection_index]
+      if @connections[@connection_index]
+	ldap = @connections[@connection_index]
+	@connection_index += 1
+	result = yield ldap
+	@connection_index -= 1
       end
       result
     end
-    
+
+
     def auth
       {:method=>bind_method, :username=>username, :password=>password}
     end
-    
+
     # Produce an array of operation arrays suitable for the LDAP library
     def to_ldap_operations operations
       operations.map {|op| [op.type, op.subject, op.values]}
     end
-    
+
   end
 end

data/lib/ruby_sync/connectors/memory_association_tracking.rb

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+ 
+
+module RubySync::Connectors::MemoryAssociationTracking
+        
+  # Returns an instance based hash association=>path
+  def paths_by_association
+    @paths_by_association ||= {}
+  end
+  
+  # Returns an instance based hash path=>{context=>key}
+  def associations_by_path
+    @assocications_by_path ||= {}
+  end
+  
+  # Store association for the given path
+  def associate association, path
+    paths_by_association[association.to_s] = path
+    associations_for(path)[association.context] = association.key
+  end
+      
+  def path_for_association association
+    paths_by_association[association.to_s]
+  end
+      
+  def associations_for path
+    associations_by_path[path] ||= {}
+  end
+
+  def remove_association association
+    path = paths_by_association[association]
+    if path
+      paths_by_association.delete(association)
+      associations_for(path).delete(association.context)
+    end
+  end
+
+  def association_key_for context, path
+    associations_for(path)[context]
+  end
+
+  def remove_associations
+    @paths_by_association = nil
+    @associations_by_path = nil
+  end
+end

data/lib/ruby_sync/connectors/memory_change_tracking.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+require 'digest/md5'
+
+# When included by a connector, tracks changes to the connector using
+# a dbm database of the form path:digest. Digest is a MD5 hash of the
+# record so we can tell if the record has changed. We can't, however,
+# tell how the record has changed so change events generated will be
+# for the entire record.
+module RubySync::Connectors::MemoryChangeTracking
+
+
+  def shadow
+    @shadow ||= {}
+  end
+
+  # Subclasses MAY override this to interface with the external system
+  # and generate an event for every change that affects items within
+  # the scope of this connector.
+  #
+  # The default behaviour is to compare a hash of each entry in the
+  # database with a stored hash of its previous value and generate
+  # add, modify and delete events appropriately. This is normally a very
+  # inefficient way to operate so overriding this method is highly
+  # recommended if you can detect changes in a more efficient manner.
+  #
+  # This method will be called repeatedly until the connector is
+  # stopped.
+  def each_change
+    # scan existing entries to see if any new or modified
+    each_entry do |path, entry|
+      digest = digest(entry)
+      unless stored_digest = shadow[path.to_s] and digest == stored_digest
+	operations = create_operations_for(entry)
+	yield RubySync::Event.add(self, path, nil, operations) 
+	shadow[path.to_s] = digest
+      end
+    end
+          
+    # scan shadow to find deleted
+    shadow.each do |key, stored_hash|
+      unless self[key]
+	yield RubySync::Event.delete(self, key)
+	shadow.delete key
+	if is_vault? and @pipeline
+	  association = association_for @pipeline.association_context, key
+	  remove_association association
+	end
+      end
+    end
+  end
+      
+  def digest(o)
+    Digest::MD5.hexdigest(o.to_yaml)
+  end
+
+  def remove_mirror
+    @shadow = nil
+  end
+      
+  def delete_from_mirror path
+    shadow.delete(path.to_s)
+  end
+
+  def update_mirror path
+    entry = self[path.to_s]
+    shadow[path.to_s] = digest(entry)
+  end
+      
+end 

data/lib/ruby_sync/connectors/memory_connector.rb

@@ -20,6 +20,9 @@ defined?(RubySync::Connectors::BaseConnector) or require 'ruby_sync/connectors/b
 module RubySync::Connectors
   class MemoryConnector < RubySync::Connectors::BaseConnector
 
+    include MemoryAssociationTracking
+    include MemoryChangeTracking
+
 
     def each_entry
       @data.each do |key, entry|

data/lib/ruby_sync/connectors/xml_connector.rb

@@ -146,9 +146,13 @@ filename "/tmp/rubysync.xml"
       unless @with_xml_invoked
         begin
           @with_xml_invoked = true
-          File.exist?(filename) or File.open(filename,'w') {|file| file.write('<entries/>')}
+	  File.exist?(filename) or File.open(filename,'w') {|file| file.write('<entries/>')}
           File.open(filename, "r") do |file|
-            file.flock(File::LOCK_EX)
+	    # flock behaves differently on 
+            # Windows. Seems to prevent us from reopening the file for
+            # writing even within the same process. Temporarily removed.
+            # TODO: add this again but only if not executing on Windows
+            #file.flock(File::LOCK_EX)
             @xml = Document.new(file)
             begin
               yield @xml

data/lib/ruby_sync/event.rb

@@ -17,11 +17,11 @@
 
 module RubySync
   
-   class Association
+  class Association
     attr_accessor :context, # many associations will share the same context
-                            # it is a function of pipeline and the client connector
-                            # to which the association applies
-                  :key      # the key is unique within the context and vault
+    # it is a function of pipeline and the client connector
+    # to which the association applies
+    :key      # the key is unique within the context and vault
     
     
     def self.delimiter; '$'; end
@@ -47,12 +47,12 @@ module RubySync
     include RubySync::Utilities
 
     attr_accessor :type,        # :delete, :add, :modify, :disassociate
-                  :source,
-                  :target,
-                  :payload,
-                  :source_path,
-                  :target_path,
-                  :association
+    :source,
+      :target,
+      :payload,
+      :source_path,
+      :target_path,
+      :association
 
     def self.force_resync source
       self.new(:force_resync, source)
@@ -105,9 +105,7 @@ module RubySync
     # Reduces the operations in this event to those that will
     # alter the target record
     def merge other
-#      other.type == :add or raise "Can only merge with add events"
-#      record = perform_operations(other.payload)
-      payload = effective_operations(@payload, other)
+      @payload = effective_operations(@payload, other)
     end
     
     # Retrieves all known values for the record affected by this event and
@@ -123,17 +121,31 @@ module RubySync
       @type = :add
     end
 
-    def convert_to_modify
+    
+    def convert_to_modify(other, filter)
       log.info "Converting '#{type}' event to modify"
+
+      # The add event contained an operation for each attribute of the source record.
+      # Therefore, we should delete any attributes in the target record that don't appear
+      # in the event.
+      affected = affected_subjects
+      other.each do |key, value|
+        if filter.include?(key) and !affected.include?(key)
+          log.info "Adding delete operation for #{key}"
+          @payload << Operation.delete(key)
+        end
+      end 
+
       @type = :modify
-      @payload.each do |op|
-        op.type = :replace
-      end
     end
           
+    # Return a list of subjects that this event affects
+    def affected_subjects
+      @payload.map {|op| op.subject}.uniq
+    end
 
     def hint
-    "(#{source.name} => #{target.name}) #{source_path}"
+      "(#{source.name} => #{target.name}) #{source_path}"
     end
           
     
@@ -163,35 +175,39 @@ module RubySync
       @uncommitted_operations = @uncommitted_operations.delete_if {|op| subjects.include? op.subject }
     end
 
+    def downcase_subjects
+      @uncommitted_operations = uncommitted_operations.map {|op| Operation.new(op.type, op.subject.downcase, op.values)}
+    end
+
     def drop_all_but_changes_to *subjects
       subjects = subjects.flatten.collect {|s| s.to_s}
       @uncommitted_operations = uncommitted_operations.delete_if {|op| !subjects.include?(op.subject.to_s)}
     end
    
-   def delete_when_blank
-       @uncommitted_operations = uncommitted_operations.map do |op| 
-         if op.sets_blank?
-	   @type == :modify ? op.same_but_as(:delete) : nil
-	 else
-	   op
-	 end
-       end.compact
-   end    
+    def delete_when_blank
+      @uncommitted_operations = uncommitted_operations.map do |op| 
+	if op.sets_blank?
+	  @type == :modify ? op.same_but_as(:delete) : nil
+	else
+	  op
+	end
+      end.compact
+    end    
        
      
-     # Add a value to a given subject unless it already sets a value
-     def add_default field_name, value
-       add_value(field_name.to_s, value) unless sets_value? field_name.to_s
-     end
+    # Add a value to a given subject unless it already sets a value
+    def add_default field_name, value
+      add_value(field_name.to_s, value) unless sets_value? field_name.to_s
+    end
      
      
-     def add_value field_name, value
-       uncommitted_operations << Operation.new(:add, field_name.to_s, as_array(value))
-     end
+    def add_value field_name, value
+      uncommitted_operations << Operation.new(:add, field_name.to_s, as_array(value))
+    end
      
-     def set_value field_name, value
-       uncommitted_operations << Operation.new(:replace, field_name.to_s, as_array(value))
-     end
+    def set_value field_name, value
+      uncommitted_operations << Operation.new(:replace, field_name.to_s, as_array(value))
+    end
 
     def values_for field_name, default=[]
       values = perform_operations @payload, {}, :subjects=>[field_name.to_s]
@@ -252,21 +268,28 @@ module RubySync
     def place(&blk)
       self.target_path = blk.call
     end
+
+    # true unless this event in its current state would have no impact
+    def effective_operation?
+      !(
+	@type == :modify && @payload.empty?
+      )
+    end
   
-  protected
+    protected
 
-  # Try to make a sensible association from the passed in object
-  def make_association o  
-    if o.kind_of? Array
-      return Association.new(o[0],o[1])
-    elsif o.kind_of? RubySync::Association
-      return o
-    elsif o
-      return Association.new(nil, o)
-    else
-      nil
-    end
-  end
+    # Try to make a sensible association from the passed in object
+    def make_association o  
+      if o.kind_of?(Array) and o.size == 2
+	return Association.new(o[0],o[1])
+      elsif o.kind_of? RubySync::Association
+	return o
+      elsif o
+	return Association.new(nil, o)
+      else
+	nil
+      end
+    end