rubysync 0.0.3 → 0.0.4

data/lib/ruby_sync/connectors/csv_file_connector.rb

@@ -17,17 +17,25 @@ module RubySync
     class CsvFileConnector < RubySync::Connectors::FileConnector
       
       option  :field_names, # A list of names representing the namesspace for this connector
-              :path_field # The name of the field to use as the source_path
+              :path_field, # The name of the field to use as the source_path
+              :header_line # true if the first line is a header and should be ignored during imports
       
       in_glob       '*.csv'
       out_extension '.csv'
       field_names   []
-      path_field    (get_field_names.empty?)? 'field_0': @field_names[0]
+      path_field((get_field_names.empty?)? 'field_0': @field_names[0])
+      
       
       # Called for each filename matching in_glob in in_path
       # Yields a modify event for each row found in the file.
       def each_file_change(filename)
+        header = header_line
         CSV.open(filename, 'r') do |row|
+          if header # should we ignore the first line
+            header = false
+            next
+          end
+
           if defined? field_name &&row.length > field_names.length
             log.warn "#{name}: Row in file #{filename} exceeds defined field_names"
           end
@@ -35,7 +43,7 @@ module RubySync
           data = {}
           row.each_index do |i|
             field_name = (i < field_names.length)? field_names[i] : "field_#{i}"
-            data[field_name] = row[i].data
+            row[i] and data[field_name] = row[i].data
           end
           association_key = source_path = path_for(data)
           yield RubySync::Event.modify(self, source_path, association_key, create_operations_for(data))
@@ -44,7 +52,11 @@ module RubySync
 
       def self.sample_config
           return <<END
-          
+
+            # True if the first line of each file is a header
+            # and should be ignored
+            header_line   true
+
             field_names   ['names', 'of', 'the', 'columns']
             path_field    'name_of_field_to_use_as_the_id'
             in_path       '/directory/to/read/files/from'

data/lib/ruby_sync/connectors/ldap_associations.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+lib_path = File.dirname(__FILE__) + '/..'
+$:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
+
+require 'ruby_sync'
+require 'ldap_connector'
+$VERBOSE = false
+require 'net/ldap'
+$VERBOSE = true
+
+
+RUBYSYNC_ASSOCIATION_ATTRIBUTE = "RubySyncAssociation"
+RUBYSYNC_ASSOCIATION_CLASS = "RubySyncSynchable"
+
+
+module LdapAssociations
+  
+
+    def associate association, path
+      with_ldap do |ldap|
+        # todo: check and warn if path is outside of search_base
+        ldap.modify :dn=>path, :operations=>[
+          [:add, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
+          ]
+      end
+    end
+    
+    def path_for_association association
+      with_ldap do |ldap|
+        filter = "#{RUBYSYNC_ASSOCIATION_ATTRIBUTE}=#{association.to_s}"
+        log.debug "Searching with filter: #{filter}"
+        results = ldap.search :base=>@search_base,
+                    :filter=>filter,
+                    :attributes=>[]
+        results or return nil
+        case results.length
+        when 0: return nil
+        when 1: return results[0].dn
+        else
+          raise Exception.new("Duplicate association found for #{association.to_s}")
+        end
+      end
+    end
+    
+    def associations_for path
+      with_ldap do |ldap|
+        results = ldap.search :base=>path,
+                    :scope=>Net::LDAP::SearchScope_BaseObject,
+                    :attributes=>[RUBYSYNC_ASSOCIATION_ATTRIBUTE]
+        unless results and results.length > 0
+          log.warn "Attempted association lookup on non-existent LDAP entry '#{path}'"
+          return []
+        end
+        associations = results[0][RUBYSYNC_ASSOCIATION_ATTRIBUTE]
+        return (associations)? as_array(associations) : []
+      end
+    end
+    
+    def remove_association association
+      path = path_for_association association
+      with_ldap do |ldap|
+        ldap.modify :dn=>path, :modifications=>[
+          [:delete, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
+          ]
+      end
+    end
+
+
+    # def associate_with_foreign_key key, path
+    #   with_ldap do |ldap|
+    #     ldap.add_attribute(path, association_attribute, key.to_s)
+    #   end
+    # end
+    # 
+    # def path_for_foreign_key key
+    #   entry = entry_for_foreign_key key
+    #   (entry)? entry.dn : nil
+    # end
+    # 
+    # def foreign_key_for path
+    #     entry = self[path]
+    #     (entry)? entry.dn : nil # TODO: That doesn't look right. Should return an association key, not a path.
+    # end
+    # 
+    # def remove_foreign_key key
+    #   with_ldap do |ldap|
+    #     entry = entry_for_foreign_key key
+    #     if entry
+    #       modify :dn=>entry.dn, :operations=>[ [:delete, association_attribute, key] ]
+    #     end
+    #   end
+    # end
+    # 
+    # def find_associated foreign_key
+    #   entry = entry_for_foreign_key key
+    #   (entry)? operations_for_entry(entry) : nil
+    # end  
+    
+    private
+    
+    def entry_for_foreign_key key
+      with_ldap do |ldap|
+        result = ldap.search :base=>search_base, :filter=>"#{association_attribute}=#{key}"
+        return nil if !result or result.size == 0
+        result[0]
+      end
+    end    
+
+end
+
+

data/lib/ruby_sync/connectors/ldap_changelog_connector.rb

@@ -0,0 +1,127 @@
+#!/usr/bin/env ruby
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+
+lib_path = File.dirname(__FILE__) + '/..'
+$:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
+
+require 'ruby_sync'
+require 'ldap_connector'
+require 'net/ldif'
+$VERBOSE = false
+require 'net/ldap'
+#$VERBOSE = true
+
+class LdapChangelogConnector < RubySync::Connectors::LdapConnector
+
+  option             :changelog_dn
+      changelog_dn          "cn=changelog"
+      
+    def initialize options={}
+      super options 
+      @last_change_number = 1
+      # TODO: Persist the current CSN, for now we'll just skip to the end of the changelog
+      skip_existing_changelog_entries
+    end
+    # Look for changelog entries. This is not supported by all LDAP servers
+    # Changelog entries have these attributes
+    # targetdn
+    # changenumber
+    # objectclass
+    # changes
+    # changetime
+    # changetype
+    # dn
+    #
+    # TODO: Detect presence/location of changelog from root DSE
+    def each_change
+      with_ldap do |ldap|
+        log.debug "@last_change_number = #{@last_change_number}"
+        filter = "(changenumber>=#{@last_change_number})"
+        first = true
+        @full_refresh_required = false
+        ldap.search :base => changelog_dn, :filter =>filter do |change|
+          change_number = change.changenumber[0].to_i
+          if first
+            first = false
+            # TODO: Persist the change_number so that we don't do a full resync everytime rubysync starts
+            if change_number != @last_change_number
+              log.warn "Earliest change number (#{change_number}) differs from that recorded (#{@last_change_number})."
+              log.warn "A full refresh is required."
+              @full_refresh_required = true
+              break
+            end
+          else
+            @last_change_number = change_number if change_number > @last_change_number
+            # todo: A proper DN object would be nice instead of string manipulation
+            target_dn = change.targetdn[0].gsub(/\s*,\s*/,',')
+            if target_dn =~ /#{search_base}$/oi
+              change_type = change.changetype[0]
+              event = event_for_changelog_entry(change)
+              yield event
+            end
+          end
+        end
+      end
+    end
+    
+    
+    def skip_existing_changelog_entries
+      with_ldap do |ldap|
+        filter = "(changenumber>=#{@last_change_number})"
+        @full_refresh_required = false
+        ldap.search :base => changelog_dn, :filter =>filter do |change|
+          change_number = change.changenumber[0].to_i
+          @last_change_number = change_number if change_number > @last_change_number
+        end
+      end
+    end
+
+    
+    # Called by unit tests to inject data
+    def test_add id, details
+      details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson', 'organizationalPerson', 'person', 'top', 'RubySyncSynchable'])
+      add id, details
+    end
+    
+    
+  private
+  
+  
+    def event_for_changelog_entry cle
+      payload = nil
+      dn = cle.targetdn[0]
+      changetype = cle.changetype[0]
+      if cle.attribute_names.include? :changes
+        payload = []
+        cr = Net::LDIF.parse("dn: #{dn}\nchangetype: #{changetype}\n#{cle.changes[0]}")[0]
+        if  changetype.to_sym == :add
+          # cr.data will be a hash of arrays or strings (attr-name=>[value1, value2, ...])
+          cr.data.each do |name, values|
+            payload << RubySync::Operation.add(name, values)
+          end
+        else
+          # cr.data will be an array of arrays of form [:action, :subject, [values]]
+          cr.data.each do |record|
+            payload << RubySync::Operation.new(record[0], record[1], record[2])
+          end
+        end
+      end
+      RubySync::Event.new(changetype, self, dn, nil, payload)
+    end
+  
+end
+
+

data/lib/ruby_sync/connectors/ldap_connector.rb

@@ -23,8 +23,6 @@ $VERBOSE = false
 require 'net/ldap'
 #$VERBOSE = true
 
-RUBYSYNC_ASSOCIATION_ATTRIBUTE = "RubySyncAssociation"
-RUBYSYNC_ASSOCIATION_CLASS = "RubySyncSynchable"
 
 class Net::LDAP::Entry
   def to_hash
@@ -36,27 +34,22 @@ module RubySync::Connectors
   class LdapConnector < RubySync::Connectors::BaseConnector
     
     option  :host,
-            :port,
-            :bind_method,
-            :username,
-            :password,
-            :search_filter,
-            :search_base,
-            :association_attribute, # name of the attribute in which to store the association key(s)
-            :changelog_dn
+    :port,
+    :bind_method,
+    :username,
+    :password,
+    :search_filter,
+    :search_base,
+    :association_attribute # name of the attribute in which to store the association key(s)
             
     association_attribute 'RubySyncAssociation'
     bind_method           :simple
     host                  'localhost'
     port                  389
     search_filter         "cn=*"
-    changelog_dn          "cn=changelog"
 
     def initialize options={}
-      super options 
-      @last_change_number = 1
-      # TODO: Persist the current CSN, for now we'll just skip to the end of the changelog
-      skip_existing_changelog_entries
+      super options
     end
 
 
@@ -64,68 +57,11 @@ module RubySync::Connectors
       #TODO: If vault, check the schema to make sure that the association_attribute is there
     end
     
-    
-    # Look for changelog entries. This is not supported by all LDAP servers
-    # you may need to subclass for OpenLDAP and Active Directory
-    # Changelog entries have these attributes
-    # targetdn
-    # changenumber
-    # objectclass
-    # changes
-    # changetime
-    # changetype
-    # dn
-    #
-    # TODO: Detect presence/location of changelog from root DSE
-    def each_change
-      with_ldap do |ldap|
-        log.debug "@last_change_number = #{@last_change_number}"
-        filter = "(changenumber>=#{@last_change_number})"
-        first = true
-        @full_refresh_required = false
-        ldap.search :base => changelog_dn, :filter =>filter do |change|
-          change_number = change.changenumber[0].to_i
-          if first
-            first = false
-            # TODO: Persist the change_number so that we don't do a full resync everytime rubysync starts
-            if change_number != @last_change_number
-              log.warn "Earliest change number (#{change_number}) differs from that recorded (#{@last_change_number})."
-              log.warn "A full refresh is required."
-              @full_refresh_required = true
-              break
-            end
-          else
-            @last_change_number = change_number if change_number > @last_change_number
-            # todo: A proper DN object would be nice instead of string manipulation
-            target_dn = change.targetdn[0].gsub(/\s*,\s*/,',')
-            if target_dn =~ /#{search_base}$/oi
-              change_type = change.changetype[0]
-              event = event_for_changelog_entry(change)
-              yield event
-            end
-          end
-        end
-      end
-    end
-    
-    
-    def skip_existing_changelog_entries
-      with_ldap do |ldap|
-        filter = "(changenumber>=#{@last_change_number})"
-        @full_refresh_required = false
-        ldap.search :base => changelog_dn, :filter =>filter do |change|
-          change_number = change.changenumber[0].to_i
-          @last_change_number = change_number if change_number > @last_change_number
-        end
-      end
-    end
 
     def each_entry
       Net::LDAP.open(:host=>host, :port=>port, :auth=>auth) do |ldap|
-        ldap.search :base => search_base, :filter => search_filter do |entry|
-          operations = operations_for_entry(entry)
-          association_key = (is_vault?)? nil : entry.dn
-          yield RubySync::Event.add(self, entry.dn, association_key, operations)
+        ldap.search :base => search_base, :filter => search_filter do |ldap_entry|
+          yield ldap_entry.dn, to_entry(ldap_entry)
         end
       end
     end
@@ -137,10 +73,6 @@ module RubySync::Connectors
       log.warn "Returning a likely sample set."
       %w{ cn givenName sn }
     end
-
-    
-    def stopped
-    end
     
 
 
@@ -148,11 +80,11 @@ module RubySync::Connectors
       return <<END
       
    host           'localhost'
-   port           10389
-   username       'uid=admin,ou=system'
+   port            389
+   username       'cn=Manager,dc=my-domain,dc=com'
    password       'secret'
    search_filter  "cn=*"
-   search_base    "dc=example,dc=com"
+   search_base    "ou=users,o=my-organization,dc=my-domain,dc=com"
    #:bind_method  :simple
 END
     end
@@ -160,12 +92,16 @@ END
 
 
     def add(path, operations)
+      result = nil
       with_ldap do |ldap|
-        ldap.add :dn=>path, :attributes=>perform_operations(operations)
+        attributes = perform_operations(operations)
+        result = ldap.add :dn=>path, :attributes=>attributes
       end
+      log.debug("ldap.add returned '#{result}'")
       return true
-    rescue Net::LdapException
-      log.warning "Exception occurred while adding LDAP record"
+    rescue Exception
+      log.warn "Exception occurred while adding LDAP record"
+      log.debug $!
       false
     end
 
@@ -192,7 +128,7 @@ END
     
     # Called by unit tests to inject data
     def test_add id, details
-      details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson', 'organizationalPerson', 'person', 'top', 'RubySyncSynchable'])
+      details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson'])
       add id, details
     end
     
@@ -201,111 +137,17 @@ END
       #is_vault? and event.add_value 'objectclass', RUBYSYNC_ASSOCIATION_CLASS
     end
 
-    def associate association, path
-      with_ldap do |ldap|
-        # todo: check and warn if path is outside of search_base
-        ldap.modify :dn=>path, :operations=>[
-          [:add, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
-          ]
-      end
-    end
-    
-    def path_for_association association
-      with_ldap do |ldap|
-        filter = "#{RUBYSYNC_ASSOCIATION_ATTRIBUTE}=#{association.to_s}"
-        log.debug "Searching with filter: #{filter}"
-        results = ldap.search :base=>@search_base,
-                    :filter=>filter,
-                    :attributes=>[]
-        results or return nil
-        case results.length
-        when 0: return nil
-        when 1: return results[0].dn
-        else
-          raise Exception.new("Duplicate association found for #{association.to_s}")
-        end
-      end
-    end
-    
-    def associations_for path
-      with_ldap do |ldap|
-        results = ldap.search :base=>path,
-                    :scope=>Net::LDAP::SearchScope_BaseObject,
-                    :attributes=>[RUBYSYNC_ASSOCIATION_ATTRIBUTE]
-        unless results and results.length > 0
-          log.warn "Attempted association lookup on non-existent LDAP entry '#{path}'"
-          return []
-        end
-        associations = results[0][RUBYSYNC_ASSOCIATION_ATTRIBUTE]
-        return (associations)? associations.as_array : []
-      end
-    end
-    
-    def remove_association association
-      path = path_for_association association
-      with_ldap do |ldap|
-        ldap.modify :dn=>path, :modifications=>[
-          [:delete, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
-          ]
-      end
-    end
-
-
-    # def associate_with_foreign_key key, path
-    #   with_ldap do |ldap|
-    #     ldap.add_attribute(path, association_attribute, key.to_s)
-    #   end
-    # end
-    # 
-    # def path_for_foreign_key key
-    #   entry = entry_for_foreign_key key
-    #   (entry)? entry.dn : nil
-    # end
-    # 
-    # def foreign_key_for path
-    #     entry = self[path]
-    #     (entry)? entry.dn : nil # TODO: That doesn't look right. Should return an association key, not a path.
-    # end
-    # 
-    # def remove_foreign_key key
-    #   with_ldap do |ldap|
-    #     entry = entry_for_foreign_key key
-    #     if entry
-    #       modify :dn=>entry.dn, :operations=>[ [:delete, association_attribute, key] ]
-    #     end
-    #   end
-    # end
-    # 
-    # def find_associated foreign_key
-    #   entry = entry_for_foreign_key key
-    #   (entry)? operations_for_entry(entry) : nil
-    # end
 
 
-private
+    private
 
-    def event_for_changelog_entry cle
-      payload = nil
-      dn = cle.targetdn[0]
-      changetype = cle.changetype[0]
-      if cle.attribute_names.include? :changes
-        payload = []
-        cr = Net::LDIF.parse("dn: #{dn}\nchangetype: #{changetype}\n#{cle.changes[0]}")[0]
-        if  changetype.to_sym == :add
-          # cr.data will be a hash of arrays or strings (attr-name=>[value1, value2, ...])
-          cr.data.each do |name, values|
-            payload << RubySync::Operation.add(name, values)
-          end
-        else
-          # cr.data will be an array of arrays of form [:action, :subject, [values]]
-          cr.data.each do |record|
-            payload << RubySync::Operation.new(record[0], record[1], record[2])
-          end
-        end
+    def to_entry ldap_entry
+      entry = {}
+      ldap_entry.each do |name, values|
+        entry[name.to_s] = values.map {|v| String.new(v)}
       end
-      RubySync::Event.new(changetype, self, dn, nil, payload)
+      entry
     end
-    
 
     def operations_for_entry entry
       ops = []
@@ -315,13 +157,8 @@ private
       ops
     end
 
-    def entry_for_foreign_key key
-      with_ldap do |ldap|
-        result = ldap.search :base=>search_base, :filter=>"#{association_attribute}=#{key}"
-        return nil if !result or result.size == 0
-        result[0]
-      end
-    end
+    
+
 
 
     def with_ldap