rubysync 0.0.1 → 0.0.2

data/lib/ruby_sync/connectors/csv_file_connector.rb

@@ -16,21 +16,17 @@ module RubySync
     # This Connector can't act as an identity vault.
     class CsvFileConnector < RubySync::Connectors::FileConnector
       
-      attr_accessor :field_names # A list of names representing the namesspace for this connector
-      attr_accessor :path_field # The name of the field to use as the source_path
+      option  :field_names, # A list of names representing the namesspace for this connector
+              :path_field # The name of the field to use as the source_path
       
-
-      def initialize options={}
-        super options
-        @in_glob ||= '*.csv'
-        @out_extension ||= '.csv'
-        @field_names ||= []
-        @path_field ||= (@field_names.empty?)? 'field_0': @field_names[0]
-      end
+      in_glob       '*.csv'
+      out_extension '.csv'
+      field_names   []
+      path_field    (get_field_names.empty?)? 'field_0': @field_names[0]
       
       # Called for each filename matching in_glob in in_path
       # Yields a modify event for each row found in the file.
-      def check_file(filename)
+      def each_file_change(filename)
         CSV.open(filename, 'r') do |row|
           if defined? field_name &&row.length > field_names.length
             log.warn "#{name}: Row in file #{filename} exceeds defined field_names"
@@ -48,14 +44,13 @@ module RubySync
 
       def self.sample_config
           return <<END
-          options(
-            :field_names=>['names', 'of', 'the', 'columns'],
-            :path_field=>'name_of_field_to_use_as_the_id',
-            :in_path=>'/directory/to/read/files/from',
-            :out_path=>'/directory/to/write/files/to',
-            :in_glob=>'*.csv',
-            :out_extension=>'.csv'
-          )
+          
+            field_names   ['names', 'of', 'the', 'columns']
+            path_field    'name_of_field_to_use_as_the_id'
+            in_path       '/directory/to/read/files/from'
+            out_path      '/directory/to/write/files/to'
+            in_glob       '*.csv'
+            out_extension '.csv'
 END
       end
             
@@ -68,7 +63,7 @@ END
 
       def write_record file, path, operations
         record = perform_operations operations
-        line = CSV.generate_line(@field_names.map {|f| record[f]})
+        line = CSV.generate_line(field_names.map {|f| record[f]})
         file.puts line
       end
         
@@ -77,8 +72,8 @@ END
       # Return the value to be used as the source_path for the event given the
       # supplied row data.
       def path_for(data)
-        if defined? @path_field
-          return data[@path_field]
+        if defined? path_field
+          return data[path_field]
         end
         return nil
       end

data/lib/ruby_sync/connectors/file_connector.rb

@@ -14,19 +14,19 @@ module RubySync
     # and/or write received events to a file.
     class FileConnector < RubySync::Connectors::BaseConnector
       
-      attr_accessor :in_path  # scan this directory for suitable files
-      attr_accessor :out_path # write received events to this directory
-      attr_accessor :out_extension # the file extension of files written to out_path
-      attr_accessor :in_glob # The filename glob for incoming files
+      option  :in_path,  # scan this directory for suitable files
+              :out_path, # write received events to this directory
+              :out_extension, # the file extension of files written to out_path
+              :in_glob # The filename glob for incoming files
       
+      out_extension     ".out"
       
       def started
-        ensure_dir_exists @in_path
-        ensure_dir_exists @out_path
-        @out_extension ||= ".out"
+        ensure_dir_exists in_path
+        ensure_dir_exists out_path
       end
       
-      def check(&blk)
+      def each_change(&blk)
         unless in_glob
           log.error "in_glob not set on file connector. No files will be processed"
           return
@@ -35,14 +35,14 @@ module RubySync
         Dir.chdir(in_path) do |path|
           Dir.glob(in_glob) do |filename|
             log.info "#{name}: Processing '#{filename}'"
-            check_file filename, &blk
+            each_file_change filename, &blk
             FileUtils.mv filename, "#{filename}.bak"
           end
         end
       end
       
       # Called for each filename matching in_glob in in_path
-      def check_file(filename,&blk)
+      def each_file_change(filename,&blk)
       end
 
 
@@ -66,7 +66,7 @@ module RubySync
       
       # Generate a unique and appropriate filename within the given path 
       def output_file_name
-       File.join(@out_path, Time.now.strftime('%Y%m%d%H%M%S') + @out_extension)
+       File.join(out_path, Time.now.strftime('%Y%m%d%H%M%S') + out_extension)
       end
             
     end

data/lib/ruby_sync/connectors/ldap_connector.rb

@@ -18,11 +18,14 @@ lib_path = File.dirname(__FILE__) + '/..'
 $:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path(lib_path))
 
 require 'ruby_sync'
-
+require 'net/ldif'
 $VERBOSE = false
 require 'net/ldap'
 #$VERBOSE = true
 
+RUBYSYNC_ASSOCIATION_ATTRIBUTE = "RubySyncAssociation"
+RUBYSYNC_ASSOCIATION_CLASS = "RubySyncSynchable"
+
 class Net::LDAP::Entry
   def to_hash
     return @myhash.dup
@@ -32,18 +35,94 @@ end
 module RubySync::Connectors
   class LdapConnector < RubySync::Connectors::BaseConnector
     
-    attr_accessor :host, :port, :bind_method, :username, :password,
-                  :search_filter, :search_base,
-                  :association_attribute # name of the attribute in which to store the association key(s)
-    
+    option  :host,
+            :port,
+            :bind_method,
+            :username,
+            :password,
+            :search_filter,
+            :search_base,
+            :association_attribute, # name of the attribute in which to store the association key(s)
+            :changelog_dn
+            
+    association_attribute 'RubySyncAssociation'
+    bind_method           :simple
+    host                  'localhost'
+    port                  389
+    search_filter         "cn=*"
+    changelog_dn          "cn=changelog"
+
+    def initialize options={}
+      super options 
+      @last_change_number = 1
+      # TODO: Persist the current CSN, for now we'll just skip to the end of the changelog
+      skip_existing_changelog_entries
+    end
+
+
     def started
       #TODO: If vault, check the schema to make sure that the association_attribute is there
-      @association_attribute ||= 'RubySyncAssociation'
     end
     
-    def check
-      Net::LDAP.open(:host=>@host, :port=>@port, :auth=>auth) do |ldap|
-        ldap.search :base => @search_base, :filter => @search_filter do |entry|
+    
+    # Look for changelog entries. This is not supported by all LDAP servers
+    # you may need to subclass for OpenLDAP and Active Directory
+    # Changelog entries have these attributes
+    # targetdn
+    # changenumber
+    # objectclass
+    # changes
+    # changetime
+    # changetype
+    # dn
+    #
+    # TODO: Detect presence/location of changelog from root DSE
+    def each_change
+      with_ldap do |ldap|
+        log.debug "@last_change_number = #{@last_change_number}"
+        filter = "(changenumber>=#{@last_change_number})"
+        first = true
+        @full_refresh_required = false
+        ldap.search :base => changelog_dn, :filter =>filter do |change|
+          change_number = change.changenumber[0].to_i
+          if first
+            first = false
+            # TODO: Persist the change_number so that we don't do a full resync everytime rubysync starts
+            if change_number != @last_change_number
+              log.warn "Earliest change number (#{change_number}) differs from that recorded (#{@last_change_number})."
+              log.warn "A full refresh is required."
+              @full_refresh_required = true
+              break
+            end
+          else
+            @last_change_number = change_number if change_number > @last_change_number
+            # todo: A proper DN object would be nice instead of string manipulation
+            target_dn = change.targetdn[0].gsub(/\s*,\s*/,',')
+            if target_dn =~ /#{search_base}$/oi
+              change_type = change.changetype[0]
+              event = event_for_changelog_entry(change)
+              yield event
+            end
+          end
+        end
+      end
+    end
+    
+    
+    def skip_existing_changelog_entries
+      with_ldap do |ldap|
+        filter = "(changenumber>=#{@last_change_number})"
+        @full_refresh_required = false
+        ldap.search :base => changelog_dn, :filter =>filter do |change|
+          change_number = change.changenumber[0].to_i
+          @last_change_number = change_number if change_number > @last_change_number
+        end
+      end
+    end
+
+    def each_entry
+      Net::LDAP.open(:host=>host, :port=>port, :auth=>auth) do |ldap|
+        ldap.search :base => search_base, :filter => search_filter do |entry|
           operations = operations_for_entry(entry)
           association_key = (is_vault?)? nil : entry.dn
           yield RubySync::Event.add(self, entry.dn, association_key, operations)
@@ -63,26 +142,18 @@ module RubySync::Connectors
     def stopped
     end
     
-    def initialize options
-      super options
-      @bind_method ||= :simple
-      @host ||= 'localhost'
-      @port ||= 389
-      @search_filter ||= "cn=*"
-    end
 
 
     def self.sample_config
       return <<END
-  options(
-   :host=>'localhost',
-   :port=>10389,
-   :username=>'uid=admin,ou=system',
-   :password=>'secret',
-   :search_filter=>"cn=*",
-   :search_base=>"dc=example,dc=com"
-  # :bind_method=>:simple,
-  )
+      
+   host           'localhost'
+   port           10389
+   username       'uid=admin,ou=system'
+   password       'secret'
+   search_filter  "cn=*"
+   search_base    "dc=example,dc=com"
+   #:bind_method  :simple
 END
     end
 
@@ -90,7 +161,7 @@ END
 
     def add(path, operations)
       with_ldap do |ldap|
-        return false unless ldap.add :dn=>path, :attributes=>perform_operations(operations)
+        ldap.add :dn=>path, :attributes=>perform_operations(operations)
       end
       return true
     rescue Net::LdapException
@@ -111,50 +182,132 @@ END
       with_ldap do |ldap|
         result = ldap.search :base=>path, :scope=>Net::LDAP::SearchScope_BaseObject, :filter=>'objectclass=*'
         return nil if !result or result.size == 0
-        result[0].to_hash
+        answer = {}
+        result[0].attribute_names.each do |name|
+          answer[name.to_s] = result[0][name]
+        end
+        answer
       end
     end
     
+    # Called by unit tests to inject data
+    def test_add id, details
+      details << RubySync::Operation.new(:add, "objectclass", ['inetOrgPerson', 'organizationalPerson', 'person', 'top', 'RubySyncSynchable'])
+      add id, details
+    end
+    
     def target_transform event
-      event.add_default 'objectclass', 'inetOrgUser'
-      # TODO: Add modifier and timestamp unless LDAP dir does this automatically
+      #event.add_default 'objectclass', 'inetOrgUser'
+      #is_vault? and event.add_value 'objectclass', RUBYSYNC_ASSOCIATION_CLASS
     end
 
-    def associate_with_foreign_key key, path
+    def associate association, path
       with_ldap do |ldap|
-        ldap.add_attribute(path, @association_attribute, key.to_s)
+        # todo: check and warn if path is outside of search_base
+        ldap.modify :dn=>path, :operations=>[
+          [:add, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
+          ]
       end
     end
     
-    def path_for_foreign_key key
-      entry = entry_for_foreign_key key
-      (entry)? entry.dn : nil
+    def path_for_association association
+      with_ldap do |ldap|
+        filter = "#{RUBYSYNC_ASSOCIATION_ATTRIBUTE}=#{association.to_s}"
+        log.debug "Searching with filter: #{filter}"
+        results = ldap.search :base=>@search_base,
+                    :filter=>filter,
+                    :attributes=>[]
+        results or return nil
+        case results.length
+        when 0: return nil
+        when 1: return results[0].dn
+        else
+          raise Exception.new("Duplicate association found for #{association.to_s}")
+        end
+      end
     end
     
-    def foreign_key_for path
-        entry = self[path]
-        (entry)? entry.dn : nil # TODO: That doesn't look right. Should return an association key, not a path.
-    end
-
-    def remove_foreign_key key
+    def associations_for path
       with_ldap do |ldap|
-        entry = entry_for_foreign_key key
-        if entry
-          modify :dn=>entry.dn, :operations=>[ [:delete, @association_attribute, key] ]
+        results = ldap.search :base=>path,
+                    :scope=>Net::LDAP::SearchScope_BaseObject,
+                    :attributes=>[RUBYSYNC_ASSOCIATION_ATTRIBUTE]
+        unless results and results.length > 0
+          log.warn "Attempted association lookup on non-existent LDAP entry '#{path}'"
+          return []
         end
+        associations = results[0][RUBYSYNC_ASSOCIATION_ATTRIBUTE]
+        return (associations)? associations.as_array : []
       end
     end
-
-    def find_associated foreign_key
-      entry = entry_for_foreign_key key
-      (entry)? operations_for_entry(entry) : nil
-    end
     
+    def remove_association association
+      path = path_for_association association
+      with_ldap do |ldap|
+        ldap.modify :dn=>path, :modifications=>[
+          [:delete, RUBYSYNC_ASSOCIATION_ATTRIBUTE, association.to_s]
+          ]
+      end
+    end
+
+
+    # def associate_with_foreign_key key, path
+    #   with_ldap do |ldap|
+    #     ldap.add_attribute(path, association_attribute, key.to_s)
+    #   end
+    # end
+    # 
+    # def path_for_foreign_key key
+    #   entry = entry_for_foreign_key key
+    #   (entry)? entry.dn : nil
+    # end
+    # 
+    # def foreign_key_for path
+    #     entry = self[path]
+    #     (entry)? entry.dn : nil # TODO: That doesn't look right. Should return an association key, not a path.
+    # end
+    # 
+    # def remove_foreign_key key
+    #   with_ldap do |ldap|
+    #     entry = entry_for_foreign_key key
+    #     if entry
+    #       modify :dn=>entry.dn, :operations=>[ [:delete, association_attribute, key] ]
+    #     end
+    #   end
+    # end
+    # 
+    # def find_associated foreign_key
+    #   entry = entry_for_foreign_key key
+    #   (entry)? operations_for_entry(entry) : nil
+    # end
+
 
 private
 
+    def event_for_changelog_entry cle
+      payload = nil
+      dn = cle.targetdn[0]
+      changetype = cle.changetype[0]
+      if cle.attribute_names.include? :changes
+        payload = []
+        cr = Net::LDIF.parse("dn: #{dn}\nchangetype: #{changetype}\n#{cle.changes[0]}")[0]
+        if  changetype.to_sym == :add
+          # cr.data will be a hash of arrays or strings (attr-name=>[value1, value2, ...])
+          cr.data.each do |name, values|
+            payload << RubySync::Operation.add(name, values)
+          end
+        else
+          # cr.data will be an array of arrays of form [:action, :subject, [values]]
+          cr.data.each do |record|
+            payload << RubySync::Operation.new(record[0], record[1], record[2])
+          end
+        end
+      end
+      RubySync::Event.new(changetype, self, dn, nil, payload)
+    end
+    
+
     def operations_for_entry entry
-      # TODO: This could probably be done better by mixing Enumerable into Entry and then calling collect
       ops = []
       entry.each do |name, values|
         ops << RubySync::Operation.add(name, values)
@@ -164,7 +317,7 @@ private
 
     def entry_for_foreign_key key
       with_ldap do |ldap|
-        result = ldap.search :base=>@search_base, :filter=>"#{@association_attribute}=#{key}"
+        result = ldap.search :base=>search_base, :filter=>"#{association_attribute}=#{key}"
         return nil if !result or result.size == 0
         result[0]
       end
@@ -173,14 +326,14 @@ private
 
     def with_ldap
       result = nil
-      Net::LDAP.open(:host=>@host, :port=>@port, :auth=>auth) do |ldap|
+      Net::LDAP.open(:host=>host, :port=>port, :auth=>auth) do |ldap|
         result = yield ldap
       end
       result
     end
     
     def auth
-      {:method=>@bind_method, :username=>@username, :password=>@password}
+      {:method=>bind_method, :username=>username, :password=>password}
     end
     
     # Produce an array of operation arrays suitable for the LDAP library
@@ -189,4 +342,4 @@ private
     end
     
   end
-end
+end

data/lib/ruby_sync/connectors/memory_connector.rb

@@ -16,23 +16,20 @@
 
 require "yaml"
 
-class Object
-  
-  # If not already an array, slip into one
-  def as_array
-    (instance_of? Array)? self : [self]
-  end
-end
 
 module RubySync::Connectors
   class MemoryConnector < RubySync::Connectors::BaseConnector
 
-    def check
+    def each_change
       while event = @events.shift
         yield event
       end
     end
 
+    def each_entry
+      # todo implement
+    end
+
     def is_echo? event
       event.sets_value?(:modifier, 'rubysync')
     end

data/lib/ruby_sync/event.rb

@@ -17,23 +17,31 @@
 
 module RubySync
   
-  class Association
+   class Association
     attr_accessor :context, # many associations will share the same context
                             # it is a function of pipeline and the client connector
                             # to which the association applies
                   :key      # the key is unique within the context and vault
     
+    
+    def self.delimiter; '$'; end
+    
     def initialize(context, key)
       @context = context
       @key = key
     end
     
     def to_s
-      "#{context}:#{key}"
+      "#{context}#{self.class.delimiter}#{key}"
     end
 
   end  
   
+  
+  # Represents a change of some type to a record in the source datastore.
+  # If the event type is :add or :modify then the payload will be an
+  # array of RubySync::Operations describing changes to the attributes of the
+  # record.
   class Event
 
     attr_accessor :type,        # delete, add, modify ...
@@ -199,7 +207,7 @@ module RubySync
       
 
 
-    # Yield to block for each operation in the payload for which the the subject is
+    # Yield to block for each operation in the payload for which the subject is
     # the specified subject
     def each_operation_on subject
       return unless payload

data/lib/ruby_sync/operation.rb

@@ -43,7 +43,7 @@ module RubySync
     remove_method :type=
     def type=(type)
       unless [:add, :delete, :replace].include? type.to_sym
-        raise Exception.new("Invalid operation type '#{value}'")
+        raise Exception.new("Invalid operation type '#{type}'")
       end
       @type = type
     end

data/lib/ruby_sync/pipelines/base_pipeline.rb

@@ -104,6 +104,12 @@ module RubySync
         end
       end
       
+      def self.in_transform &blk
+        define_method :in_transform do |event|
+          event.meta_def :transform, &blk
+          event.transform
+        end
+      end
       
       
       # Called by the identity-vault connector in the 'out' thread to process events generated

data/lib/ruby_sync/util/utilities.rb

@@ -108,8 +108,27 @@ module RubySync
       end
       return false
     end
-        
-
-        
+       
+    # Make and instance method _name_ that returns the value set by the
+    # class method _name_.
+    # def self.class_option name
+    #   self.class_eval "def #{name}() self.class.instance_variable_get :#{name}; end"
+    #   self.instance_eval "def #{name}(value) @#{name}=value; end"
+    # end
+    
+    def get_preference(name, file_name=nil)
+      class_name ||= get_preference_file
+    end
+    
+    def set_preference(name)
+      
+    end
+    
+    def get_preference_file_path name
+      dir = "#{ENV[HOME]}/.rubysync"
+      Dir.mkdir(dir)
+      "#{dir}#{file}"
+    end
+    
   end
 end

data/lib/ruby_sync.rb

@@ -19,14 +19,20 @@ $:.unshift lib_path unless $:.include?(lib_path) || $:.include?(File.expand_path
 require 'rubygems'
 require 'active_support'
 require 'ruby_sync/util/utilities'
+require 'ruby_sync/util/metaid'
 #require 'ruby_sync/connectors/base_connector'
 #require 'ruby_sync/pipelines/base_pipeline'
 require 'ruby_sync/operation'
 require 'ruby_sync/event'
 
-# Make the log method globally available
 class Object
 
+  # If not already an array, slip into one
+  def as_array
+    (instance_of? Array)? self : [self]
+  end
+
+  # Make the log method globally available
   def log
     unless defined? @@log
       @@log = Logger.new(STDOUT)
@@ -35,7 +41,24 @@ class Object
     end
     @@log
   end
-end  
+end
+
+class Module
+  # Add an option that will be defined by a class method, stored in a class variable
+  # and accessible as an instance method
+  def option *names
+    names.each do |name|
+      meta_def name do |value|
+        class_def name do
+          value
+        end
+        meta_def "get_#{name}" do
+          value
+        end
+      end
+    end
+  end
+end
 
 class Configuration
 

data/test/data/example1.ldif

@@ -0,0 +1,20 @@
+version: 1
+dn: cn=Barbara Jensen, ou=Product Development, dc=airius, dc=com
+objectclass: top
+objectclass: person
+objectclass: organizationalPerson
+cn: Barbara Jensen
+cn: Barbara J Jensen
+cn: Babs Jensen
+sn: Jensen
+uid: bjensen
+telephonenumber: +1 408 555 1212
+description: A big sailing fan.
+
+dn: cn=Bjorn Jensen, ou=Accounting, dc=airius, dc=com
+objectclass: top
+objectclass: person
+objectclass: organizationalPerson
+cn: Bjorn Jensen
+sn: Jensen
+telephonenumber: +1 408 555 1212

data/test/data/example2.ldif

@@ -0,0 +1,14 @@
+version: 1
+dn:cn=Barbara Jensen, ou=Product Development, dc=airius, dc=com
+objectclass:top
+objectclass:person
+objectclass:organizationalPerson
+cn:Barbara Jensen
+cn:Barbara J Jensen
+cn:Babs Jensen
+sn:Jensen
+uid:bjensen
+telephonenumber:+1 408 555 1212
+description:Babs is a big sailing fan, and travels extensively in sea
+ rch of perfect sailing conditions.
+title:Product Manager, Rod and Reel Division