rubysync 0.0.1 → 0.0.2

data/bin/rubysync

@@ -224,14 +224,14 @@ puts <<"END"
     connectors/my_db_connector.rb    ;how to connect to your DB or Rails app.
 
   And enter:
-    $ rubysync pipeline my_pipeline -C my_csv -V my_db
+    $ rubysync pipeline my -C my_csv -V my_db
 
-  You would then edit the file +pipelines/my_pipeline.rb+ to configure the
+  You would then edit the file pipelines/my_pipeline.rb to configure the
   policy for synchronizing between the two connectors.
                                         
   You may then execute the pipeline in one-shot mode (daemon mode is coming):
 
-    $ rubysync once my_pipeline
+    $ rubysync once my
 END
 end
 

data/examples/ims2/connectors/hr_db_connector.rb

@@ -1,8 +1,6 @@
 class HrDbConnector < RubySync::Connectors::ActiveRecordConnector
-            options(
-            :name=>"HR Database",
-            :application=>"#{File.dirname(__FILE__)}/../../ar_webapp",
-            :model=>:person
-          )
+  name        "HR Database",
+  application "#{File.dirname(__FILE__)}/../../ar_webapp",
+  model       :person
 
 end

data/examples/my_ims/connectors/my_csv_connector.rb

@@ -0,0 +1,10 @@
+class MyCsvConnector < RubySync::Connectors::CsvFileConnector
+            
+            field_names   ['id', 'given_name', 'surname']
+            path_field    :id
+            in_path       '/tmp/rubysync/in'
+            out_path      '/tmp/rubysync/out'
+            in_glob       '*.csv'
+            out_extension '.csv'
+
+end

data/examples/my_ims/connectors/my_db_connector.rb

@@ -0,0 +1,7 @@
+class MyDbConnector < RubySync::Connectors::ActiveRecordConnector
+  
+    application "#{File.dirname(__FILE__)}/../../ar_webapp"
+    model 'person'
+
+
+end

data/examples/my_ims/pipelines/{finance_pipeline.rb → my_pipeline.rb}

@@ -1,11 +1,11 @@
-class FinancePipeline < RubySync::Pipelines::BasePipeline
+class MyPipeline < RubySync::Pipelines::BasePipeline
 
-  client :finance
+  client :my_csv
 
-  vault :hr_db
+  vault :my_db
 
   # Remove any fields that you don't want to set in the client from the vault
-  allow_out :first_name
+  allow_out :first_name, :last_name
 
   # Remove any fields that you don't want to set in the vault from the client
   allow_in :first_name, :last_name
@@ -16,17 +16,17 @@ class FinancePipeline < RubySync::Pipelines::BasePipeline
   #    'first name' => 'givenName'
   # separate each mapping with a comma.
   # The following fields were detected on the client:
-  # 'username', 'name', 'email'
+  # 'id', 'given_name', 'surname'
   map_vault_to_client(
-    'first_name' => 'name'
-		#'last_name' => 'a_client_field'
+    'first_name' => 'given_name',
+		'last_name' => 'surname'
   )
 
-  # in means going from client to vault
+  # "in" means going from client to vault
   #in_transform do
   #end
 
-  # out means going from vault to client
+  # "out" means going from vault to client
   #out_transform do
   #end
 

data/lib/net/ldif.rb

@@ -0,0 +1,302 @@
+#!/usr/bin/env ruby -w
+#
+#  Copyright (c) 2007 Ritchie Young. All rights reserved.
+#
+# This file is part of RubySync.
+# 
+# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
+# 
+# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
+
+require 'base64'
+
+module Net
+
+  class ParsingError < StandardError; end
+    
+
+  
+  # Represents a mod-spec structure from RFC2849
+  class LDIFModSpec # :nodoc:
+    attr_accessor :type, :attribute, :values
+    def initialize(type, attribute)
+      self.type = type
+      self.attribute = attribute
+      self.values = nil
+    end
+
+    def add_change name, value
+      if @values
+        if @values.kind_of? Array
+          @values << value
+        else
+          @values = [@values, value]
+        end
+      else
+        @values = value
+      end
+    end
+
+    def changes() [@type.to_sym, @attribute, @values] end
+  end
+
+  # Represents an LDIF change record as defined in RFC2849.
+  # The RFC specifies that an LDIF file can contain either
+  # attrval records (which are just content) or change records
+  # which represent changes to the content.
+  # This parser puts both types into the change record structure
+  # Ldif attrval records simply become change records of type
+  # 'add'.
+  class ChangeRecord
+    attr_accessor :dn, :changetype, :data
+    
+    def initialize(dn, changetype='add')
+      self.dn = dn
+      self.changetype = changetype
+      @mod_spec = nil
+      @data = nil
+    end
+    
+    def to_s
+      "#{@dn}\n#{@changetype}\n" +
+      @data.inspect
+    end
+      
+    
+    def add_value(name, value) # :nodoc:
+      # Changetype specified before any other fields
+      if name == 'changetype' and !@data
+        @changetype = value
+        return
+      end
+
+      if name == '-' and @changetype != 'modify'
+          raise ParsingError.new("'-' is only valid in LDIF change records when changetype is modify")
+      end
+      
+      # Just an ordinary name value pair
+      case changetype
+      when 'add': add_content(name, value)
+      when 'delete':  
+        raise ParsingError.new("Didn't expect content when changetype was 'delete', (#{name}:#{value})")
+      when 'modify': add_modification(name, value)
+      when 'modrdn': add_moddn_value(name,value)
+      when 'moddn': add_moddn_value(name, value)
+      else
+        raise ParsingError.new("Unknown changetype: '#{changetype}'")
+      end
+    end
+
+    
+    def add_modification(name, value)
+      @data ||= []
+      if name == '-'
+        @mod_spec or
+          raise ParsingError.new("'-' in LDIF modify record before any actual changes")
+        @data << @mod_spec.changes
+        @mod_spec = nil
+        return
+      end
+      
+      if @mod_spec
+        @mod_spec.add_change name,value
+      elsif %w{add delete replace}.include? name
+        @mod_spec = LDIFModSpec.new(name, value)
+      end
+    end
+    
+    def add_moddn_value(name, value)
+      #TODO: implement
+      #raise Exception.new("Sorry, not yet implemented")
+    end
+
+    def add_content(key, value)
+      @data ||= {}
+      if @data[key]
+        if @data[key].kind_of? Array
+          @data[key] << value
+        else
+          @data[key] = [@data[key], value]
+        end
+      else
+        @data[key] = value
+      end
+    end
+
+  end # of class ChangeRecord
+
+
+  class URLForValue < String # :nodoc:
+    def to_s
+      filename = self.sub(/^file:\/\//oi, '')
+      File.read(filename)
+    end
+  end
+
+  class Base64EncodedString < String # :nodoc:
+    def to_s() Base64.decode64 self; end
+  end
+
+  class LDIF
+
+    #FILL = '\s*'
+    # TODO: Attribute description should be more structured than this
+    ATTRIBUTE_DESCRIPTION = '[a-zA-Z0-9.;-]+'
+    SAFE_INIT_CHAR = '[\x01-\x09\x0b-\x0c\x0e-\x1f\x21-\x39\x3b\x3d-\x7f]'
+    SAFE_CHAR = '[\x01-\x09\x0b-\x0c\x0e-\x7f]'
+    SAFE_STRING = "#{SAFE_INIT_CHAR}#{SAFE_CHAR}*"
+    BASE64_STRING = '[\x2b\x2f\x30-\x39\x3d\x41-\x5a\x61-\x7a]*'
+
+
+    # Yields Net::ChangeRecord for each LDIF record in the file.
+    # If the file contains attr-val (content) records, they are
+    # yielded as Net::ChangeRecords of type 'add'.
+    # If no block is given, then returns an array of the
+    # Net::ChangeRecord objects.
+    def self.parse(stream)
+      return parse_to_array(stream) unless block_given?
+      type = nil
+      record_number = 0
+      record = nil
+      tokenize(stream) do |name, value|
+
+        # version-spec
+        if (name == 'version' and record_number == 0)
+          value == '1' or raise ParsingError.new("Don't know how to parse LDIF version '#{value}'")
+          next
+        end
+        
+        # Blank line
+        # Unless I'm reading the spec wrong, blank lines don't seem to mean much
+        # Yet in all the examples, the records seem to be separated by blank lines.
+        # TODO: Check whether blank lines mean anything
+        next if (name == nil)
+
+        name.downcase!        
+        # DN - start a new record
+        if name == 'dn'
+          # Process existing record
+          yield(record) if record
+          record = ChangeRecord.new(value)
+          next
+        end
+        
+        record or raise ParsingException.new("Expecting a dn, got #{name}: #{value}")        
+        record.add_value name, value
+      end # of tokens
+      yield(record) if record
+    end
+
+    def self.parse_to_array(stream)
+      changes = []
+      parse(stream) do |change|
+        changes << change
+      end
+      changes
+    end
+
+    # Yields a series of pairs of the form name, value found in the
+    # given stream. Comments (lines starting with #) are removed,
+    # base64 values are decoded and folded lines are unfolded.
+    # Blank lines are yielded with a nil name, nil value pair.
+    # Lines containing only a hyphen are yielded as a name="-",
+    # value="-" pair.
+    # Values specified as file:// urls as described in RFC2849 are
+    # replaced with the contents of the specified file. 
+    def self.tokenize(stream)
+
+      foldable = false
+      comment = false
+      name = nil
+      value = ""
+      line_number = 0
+      stream.each_line do |line|
+        line_number += 1
+        
+        # Blank line
+        if line.strip.length == 0
+          yield(name, value.to_s) if name;name = nil;value = ""
+          yield nil,nil
+          foldable = false
+          comment = false
+          next
+        end
+        
+        # Line extension
+        if foldable and line[0,1] == ' '
+          value << line.chop[1..-1] unless comment
+          next
+        end
+                    
+        # Comment
+        if line[0,1] == '#'
+          yield(name, value.to_s) if name;name = nil;value = ""
+          comment = true
+          foldable = true
+          next
+        end
+
+        # Base64 Encoded name:value pair
+        if line =~ /^(#{ATTRIBUTE_DESCRIPTION})::\s*(#{BASE64_STRING})/oi
+          yield(name, value.to_s) if name
+          name  = $1
+          value = Base64EncodedString.new($2)
+          comment = false
+          foldable = false # It is but we've got a separate rule for it
+          next
+        end
+        
+        # URL value
+        if line =~ /^(#{ATTRIBUTE_DESCRIPTION}):<\s*(#{SAFE_STRING})/oi
+            yield(name, value.to_s) if name
+            name  = $1
+            value = URLForValue.new($2)
+            comment = false
+            foldable = true
+            next
+          end
+        
+        # Name:Value pair
+        if line =~ /^(#{ATTRIBUTE_DESCRIPTION}):\s*(#{SAFE_STRING})/oi
+          yield(name, value.to_s) if name
+          name = $1; value = $2
+          foldable = true
+          comment = false
+          next
+        end
+        
+        # Hyphen
+        if line =~ /^-/o
+          yield(name, value.to_s) if name;name = nil;value = ""
+          yield('-','-')
+          foldable = false
+          comment = false
+          next
+        end
+
+        # Continuation of Base64 Encoded value?
+        if value.kind_of?(Base64EncodedString) and line =~ /^ (#{BASE64_STRING})/oi
+          value << $1
+          next
+        end
+
+        raise ParsingError.new("Unexpected LDIF at line: #{line_number}")
+      end # of file
+      yield(name, value.to_s) if name
+      line_number
+    end
+    
+    private
+    
+    
+    def process(record, type) # :nodoc:
+      
+    end
+    
+  end
+end

data/lib/ruby_sync/connectors/active_record_connector.rb

@@ -27,27 +27,29 @@ module RubySync::Connectors
   class ActiveRecordConnector < RubySync::Connectors::BaseConnector
 
 
-    attr_accessor :ar_class, :model, :application, :rails_env, :db_type, :db_host, :db_name
-    
+    option :ar_class, :model, :application, :rails_env, :db_type, :db_host, :db_name, :db_config
+    rails_env 'development'
+    db_type 'mysql'
+    db_host 'localhost'
+    db_name "rubysync_#{get_rails_env}"
+    # Default db_config in case we're not sucking the config out of a rails app
+    db_config(
+      :adapter=>get_db_type,
+      :host=>get_db_host,
+      :database=>get_db_name
+    )
+    model :user
+
+
     def initialize options={}
       super options
-      @rails_env ||= 'development'
-      @db_type ||= 'mysql'
-      @db_host ||= 'localhost'
-      @db_name ||= "rubysync_#{@rails_env}"
-      # Default db_config in case we're not sucking the config out of a rails app
-      @db_config = {
-        :adapter=>@db_type,
-        :host=>@db_host,
-        :database=>@db_name
-      }
 
       # Rails app specified, use it to configure
-      if defined? @application
+      if application
         # Load the database configuration
-        rails_app_path = File.expand_path(@application, File.dirname(__FILE__))
+        rails_app_path = File.expand_path(application, File.dirname(__FILE__))
         db_config_filename = File.join(rails_app_path, 'config', 'database.yml')
-        @db_config = YAML::load(ERB.new(IO.read(db_config_filename)).result)[@rails_env]
+        db_config = YAML::load(ERB.new(IO.read(db_config_filename)).result)[rails_env]
         # Require the models
         log.debug "Loading the models for #{self.class.name}:"
         Dir.chdir(File.join(rails_app_path,'app','models')) do
@@ -56,13 +58,12 @@ module RubySync::Connectors
             require filename
             class_name = filename[0..-4].camelize
             klass = class_name.constantize
-            klass.establish_connection @db_config if defined? klass.establish_connection
+            klass.establish_connection db_config if defined? klass.establish_connection
           end
         end
       end
 
-      @model ||= :user
-      @ar_class ||= @model.to_s.camelize.constantize
+      self.class.ar_class model.to_s.camelize.constantize
     end
 
       
@@ -73,16 +74,16 @@ module RubySync::Connectors
       
       def self.sample_config
           return <<END
-          options(
-            #:application=>'/path/to/a/rails/application',
-            #:model=>'name_of_model_to_sync'
-          )
+
+    application '/path/to/a/rails/application'
+    model 'name_of_model_to_sync'
+
 END
       end
       
       
     # Process each RubySyncEvent and then delete it from the db.
-    def check
+    def each_change
       ::RubySyncEvent.find(:all).each do |rse|
         event = RubySync::Event.new(rse.event_type, self, rse.trackable_id, nil, to_payload(rse))
         yield event
@@ -102,7 +103,7 @@ END
     # the id on creation.
     def perform_add event
       log.info "Adding '#{event.target_path}' to '#{name}'"
-      @ar_class.new() do |record|
+      ar_class.new() do |record|
         populate(record, perform_operations(event.payload))
         log.info(record.inspect)
         record.save!
@@ -119,21 +120,21 @@ END
 
       
     def modify(path, operations)
-      @ar_class.find(path) do |record|
+      ar_class.find(path) do |record|
         populate(record, perform_operations(operations))
         record.save
       end
     end
     
     def delete(path)
-      @ar_class.destroy path
+      ar_class.destroy path
     end
   
     # Implement vault functionality
 
     def associate association, path
       log.debug "Associating '#{association}' with '#{path}'"
-      ruby_sync_association.create :synchronizable_id=>path, :synchronizable_type=>@ar_class.name,
+      ruby_sync_association.create :synchronizable_id=>path, :synchronizable_type=>ar_class.name,
                                    :context=>association.context, :key=>association.key
     end
 
@@ -147,12 +148,12 @@ END
     end
 
     def association_key_for context, path
-      record = ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type_and_context path, @model.to_s, context
+      record = ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type_and_context path, model.to_s, context
       record and record.key
     end
     
     def associations_for(path)
-      ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type(path, @model.to_s)
+      ruby_sync_association.find_by_synchronizable_id_and_synchronizable_type(path, model.to_s)
     rescue ActiveRecord::RecordNotFound
       return nil
     end
@@ -165,7 +166,7 @@ END
 
 
     def [](path)
-      @ar_class.find(path)
+      ar_class.find(path)
     rescue ActiveRecord::RecordNotFound
       return nil
     end
@@ -175,13 +176,13 @@ private
     def ruby_sync_association
       unless @ruby_sync_association
         @ruby_sync_association = ::RubySyncAssociation
-        ::RubySyncAssociation.establish_connection(@db_config)
+        ::RubySyncAssociation.establish_connection(db_config)
       end
       @ruby_sync_association
     end
 
     def populate record, content
-      @ar_class.content_columns.each do |c|
+      ar_class.content_columns.each do |c|
         record[c.name] = content[c.name][0] if content[c.name]
       end
     end

data/lib/ruby_sync/connectors/base_connector.rb

@@ -23,7 +23,6 @@ module RubySync::Connectors
       
       attr_accessor :once_only, :name, :is_vault
       
-      
       def initialize options={}
         options = self.class.default_options.merge(options)
         once_only = false
@@ -52,18 +51,24 @@ module RubySync::Connectors
       def started; end
       
       # Subclasses must override this to
-      # interface with the external system and generate events.
+      # interface with the external system and generate events for every
+      # entry in the scope.
       # These events are yielded to the passed in block to process.
       # This method will be called repeatedly until the connector is
       # stopped.
-      def check; end
+      def each_entry; end
+
+      # Subclasses must override this to interface with the external system
+      # and generate an event for every change that affects items within
+      # the scope of this connector.
+      def each_change; end
       
       # Override this to perform actions that must be performed when
       # the connector exits (eg closing network conections).
       def stopped; end
 
 
-      # Call check repeatedly (or once if in once_only mode)
+      # Call each_change repeatedly (or once if in once_only mode)
       # to generate events.
       # Should generally only be called by the pipeline to which it is attached.
       def start
@@ -71,7 +76,7 @@ module RubySync::Connectors
         @running = true
         started()
         while @running
-          check do |event|
+          each_change do |event|
             if is_delete_echo?(event) || is_echo?(event)
               log.debug "Ignoring echoed event"
             else
@@ -156,9 +161,7 @@ module RubySync::Connectors
         defined? associations_for
       end
 
-      # TODO: These method signatures need to change to include a connector or pipeline id so that
-      # we can distinguish between foreign keys for the same record but different
-      # connectors/pipelines.
+      # Implement these if you want your connector to be able to act as a vault
 
       # def associate association, path
       # end
@@ -166,14 +169,22 @@ module RubySync::Connectors
       # def path_for_association association
       # end
       # 
-      # def association_key_for context, path
-      # end
       #
       # def associations_for path
       # end
       #
       # def remove_association association
       # end
+
+
+      def association_key_for context, path
+        raise "#{name} is not a vault." unless is_vault?
+        associations_for(path).each do |assoc|
+          (c, key) = assoc.split(RubySync::Association.delimiter, 2)
+          return key if c == context 
+        end
+        return nil
+      end
       
       # Return the association object given the association context and path.
       # This should only be called on the vault.