rubysync 0.0.3 → 0.0.4

data/docs/schema/99rubysync.ldif

@@ -0,0 +1,27 @@
+#
+################################################################################
+#
+dn: cn=schema
+#
+################################################################################
+#
+attributeTypes: (
+  1.3.6.1.4.1.28955.50.1.1
+  NAME 'RubySyncAssociation'
+  DESC 'Context:Key provided by connected system'
+  EQUALITY caseExactMatch
+  SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+  )
+#
+################################################################################
+#
+objectClasses: (
+  1.3.6.1.4.1.28955.50.2.1
+  NAME 'RubySyncSynchable'
+  DESC 'Object can preserve links to other objects via RubySync'
+  AUXILIARY
+  MAY RubySyncAssociation
+  )
+#
+################################################################################
+#

data/docs/schema/rubysync.schema

@@ -0,0 +1,16 @@
+# rubysync
+# Generated By LDAPStudio on : Mar 10, 2007 10:49:43 PM
+
+attributetype ( 1.3.6.1.4.1.28955.1.1.1
+	NAME 'RubySyncAssociation'
+	DESC 'Context:Key provided by connected system'
+	EQUALITY caseExactMatch
+	SYNTAX 1.3.6.1.4.1.1466.115.121.1.15
+ )
+
+objectclass ( 1.3.6.1.4.1.28955.1.2.1 NAME 'RubySyncSynchable'
+         DESC 'Object can preserve links to other objects via RubySync'
+         AUXILIARY
+         MAY RubySyncAssociation )
+
+

data/docs/to_sync.txt

@@ -0,0 +1,15 @@
+Things to Sync
+
+gmail address book
+yahoo address book
+msn address book
+delicious links
+dobot.net tasks
+palm pilot tasks
+palm address book
+windows mobile address book
+rss feeds
+any HTML page using scRubyt
+imdb.com
+
+

data/docs/walkthru.txt

@@ -0,0 +1,186 @@
+$ mkdir x
+$ cd x
+$ rubysync create kaos
+$ ls -l
+total 0
+drwxr-xr-x   7 ritchiey  ritchiey  238 Aug 21 16:47 kaos
+$ cd kaos
+$ ls -l
+total 0
+drwxr-xr-x   2 ritchiey  ritchiey   68 Aug 21 16:47 connectors
+drwxr-xr-x   2 ritchiey  ritchiey   68 Aug 21 16:47 db
+drwxr-xr-x   2 ritchiey  ritchiey   68 Aug 21 16:47 log
+drwxr-xr-x   2 ritchiey  ritchiey   68 Aug 21 16:47 pipelines
+drwxr-xr-x   5 ritchiey  ritchiey  170 Aug 21 16:47 shared
+$ rubysync connector hr -t csv_file
+$ rubysync connector kaos_vault -t xml$ mate .. 
+$ cat connectors/hr_connector.rb 
+class HrConnector < RubySync::Connectors::CsvFileConnector
+            
+            field_names   ['id', 'first_name', 'last_name', 'skills']
+            path_field    'id'
+            in_path       '/Users/ritchiey/x/in'
+            out_path      '/Users/ritchiey/x/out'
+            in_glob       '*.csv'
+            out_extension '.csv'
+
+end
+$ cat connectors/kaos_vault_connector.rb 
+class KaosVaultConnector < RubySync::Connectors::XmlConnector
+            #
+          # "filename" should be the full name of the file containing
+          # the xml representation of the synchronized content.
+          # You probably want to change this:
+          #
+          filename "/Users/ritchiey/x/kaos.xml"
+
+end
+$ rubysync fields hr
+id
+first_name
+last_name
+skills
+$ rubysync fields kaos_vault
+
+$ rubysync pipeline hr_import -C hr -V kaos_vault
+$ # Now edit the pipeline config
+$ # Actually, here's the pipeline before editing
+$ cat pipelines/hr_import_pipeline.rb 
+class HrImportPipeline < RubySync::Pipelines::BasePipeline
+
+  client :hr
+
+  vault :kaos_vault
+
+  # Remove any fields that you don't want to set in the client from the vault
+  allow_out :allow, :these, :fields, :through
+
+  # Remove any fields that you don't want to set in the vault from the client
+  allow_in :allow, :these, :fields, :through
+
+  # If the client and vault have different names for the same field, define the
+  # the mapping here. For example, if the vault has a field called "first name" and
+  # the client has a field called givenName you may put:
+  #    'first name' => 'givenName'
+  # separate each mapping with a comma.
+  # The following fields were detected on the client:
+  # 'id', 'first_name', 'last_name', 'skills'
+  map_vault_to_client({
+    #'allow' => 'a_client_field',
+                #'these' => 'a_client_field',
+                #'fields' => 'a_client_field',
+                #'through' => 'a_client_field'
+  })
+
+  # "in" means going from client to vault
+  #in_transform do
+  #end
+
+  # "out" means going from vault to client
+  #out_transform do
+  #end
+
+end
+$ # now edit the pipeline
+$ cat pipelines/hr_import_pipeline.rb 
+class HrImportPipeline < RubySync::Pipelines::BasePipeline
+
+  client :hr
+
+  vault :kaos_vault
+
+  # Remove any fields that you don't want to set in the client from the vault
+  allow_out :id, :first_name, :last_name
+
+  # Remove any fields that you don't want to set in the vault from the client
+  allow_in :id, :first_name, :last_name
+
+  # If the client and vault have different names for the same field, define the
+  # the mapping here. For example, if the vault has a field called "first name" and
+  # the client has a field called givenName you may put:
+  #    'first name' => 'givenName'
+  # separate each mapping with a comma.
+  # The following fields were detected on the client:
+  # 'id', 'first_name', 'last_name', 'skills'
+  map_vault_to_client({
+    #'allow' => 'a_client_field',
+                #'these' => 'a_client_field',
+                #'fields' => 'a_client_field',
+                #'through' => 'a_client_field'
+  })
+
+  # "in" means going from client to vault
+  #in_transform do
+  #end
+
+  # "out" means going from vault to client
+  #out_transform do
+  #end
+
+end
+$ # so all we've done specified which fields to allow in an out. Notice that we left "skills" out.
+$ # also note that we're exporting as well as importing for now
+$ # so now we can run our sync      
+$ rubysync once hr_import
+$ # oops. Forgot to give it any data
+$ # notice, though how it created the import and export directories for us.
+$ cd ../in
+$ # lets make some henchmen
+$ cat > henchmen.csv
+bobby,BareKnuckle,Bobby,pugilism:yoga
+tt,Testy,Terry,kidnapping:interrogation:juggling
+$ ls -l
+total 8
+-rw-r--r--   1 ritchiey  ritchiey  87 Aug 21 17:11 henchmen.csv
+$ # we need to run rubysync from within the configuration directory
+$ cd ../kaos/
+$ rubysync once hr_import
+$ ls -l ../in
+total 8
+-rw-r--r--   1 ritchiey  ritchiey  87 Aug 21 17:11 henchmen.csv.bak
+$ # note that the csv has been renamed
+$ # lets have a look at the xml file thats been created
+$ cat ../kaos.xml
+<opt>
+  <tt>
+    <id>tt</id>
+    <first_name>Testy</first_name>
+    <last_name>Terry</last_name>
+  </tt>
+  <bobby>
+    <id>bobby</id>
+    <first_name>BareKnuckle</first_name>
+    <last_name>Bobby</last_name>
+  </bobby>
+</opt>
+$ # ok, great. We've got a couple of records in there. 
+$ # remember that we also configured an output directory
+$ ls -l ../out
+$ # no output. ok what if we modify the vault itself
+$ cat ../kaos.xml 
+<opt>
+  <tt>
+    <id>tt</id>
+    <first_name>Testy</first_name>
+    <last_name>Terry</last_name>
+  </tt>
+  <bobby>
+    <id>bobby</id>
+    <first_name>BareKnuckle</first_name>
+    <last_name>Bobby</last_name>
+  </bobby>
+  <desd>
+    <id>desd</id>
+    <first_name>Dangerous</first_name>
+    <last_name>Des</last_name>
+  </desd>
+</opt>
+$ # and run it again
+$ rubysync once hr_import
+$ ls -l ../out
+total 8
+-rw-r--r--   1 ritchiey  ritchiey  20 Aug 21 17:25 20070821172521.csv
+$ cat ../out/20070821172521.csv 
+desd,Dangerous,Des,
+$ # So it just sends through the changes to the file 
+$ # How's it doingr

data/lib/ruby_sync.rb

@@ -21,28 +21,9 @@ require 'rubygems'
 require 'active_support'
 require 'ruby_sync/util/utilities'
 require 'ruby_sync/util/metaid'
-#require 'ruby_sync/connectors/base_connector'
-#require 'ruby_sync/pipelines/base_pipeline'
 require 'ruby_sync/operation'
 require 'ruby_sync/event'
 
-class Object
-
-  # If not already an array, slip into one
-  def as_array
-    (instance_of? Array)? self : [self]
-  end
-
-  # Make the log method globally available
-  def log
-    unless defined? @@log
-      @@log = Logger.new(STDOUT)
-      #@@log.level = Logger::DEBUG
-      @@log.datetime_format = "%H:%M:%S"
-    end
-    @@log
-  end
-end
 
 class Module
   # Add an option that will be defined by a class method, stored in a class variable
@@ -66,17 +47,17 @@ class Configuration
   include RubySync::Utilities
 
   def initialize
-    include_in_search_path "#{base_path}/pipelines"
-    include_in_search_path "#{base_path}/connectors"
-    include_in_search_path "#{base_path}/shared/connectors"
-    include_in_search_path "#{base_path}/shared/pipelines"    
-    include_in_search_path "#{base_path}/shared/lib"
+    base_path and include_in_search_path "#{base_path}/pipelines"
+    base_path and include_in_search_path "#{base_path}/connectors"
+    base_path and include_in_search_path "#{base_path}/shared/connectors"
+    base_path and include_in_search_path "#{base_path}/shared/pipelines"    
+    base_path and include_in_search_path "#{base_path}/shared/lib"
 
     lib_path = File.dirname(__FILE__)
     require_all_in_dir "#{lib_path}/ruby_sync/connectors", "*_connector.rb"
-    require_all_in_dir "#{base_path}/shared/connectors", "*_connector.rb"
+    base_path and require_all_in_dir "#{base_path}/shared/connectors", "*_connector.rb"
     require_all_in_dir "#{lib_path}/ruby_sync/pipelines", "*_pipeline.rb"
-    require_all_in_dir "#{base_path}/shared/pipelines", "*_pipeline.rb"
+    base_path and require_all_in_dir "#{base_path}/shared/pipelines", "*_pipeline.rb"
   end
 
   # We find the first directory in the search path that is a parent of the specified
@@ -108,6 +89,3 @@ class Configuration
 end
 
 Configuration.new
-
-
-  

data/lib/ruby_sync/connectors/base_connector.rb

@@ -14,7 +14,8 @@
 # Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
 
 require 'ruby_sync/connectors/connector_event_processing'
-require 'dbm'
+require 'yaml'
+require 'yaml/dbm'
 require 'digest/md5'
 
 module RubySync::Connectors
@@ -23,11 +24,15 @@ module RubySync::Connectors
       include RubySync::Utilities
       include ConnectorEventProcessing
       
-      attr_accessor :once_only, :name, :is_vault
+      attr_accessor :once_only, :name, :is_vault, :pipeline
       option  :dbm_path
       
       # set a default dbm path
-      def dbm_path() "#{base_path}/db/#{name}"; end
+      def dbm_path()
+        p = "#{base_path}/db"
+        ::FileUtils.mkdir_p p
+        ::File.join(p,name)
+      end
 
       # Stores association keys indexed by path:association_context
       def path_to_association_dbm_filename
@@ -53,7 +58,7 @@ module RubySync::Connectors
         self.name = options[:name]
         self.is_vault = options[:is_vault]
         if is_vault && !can_act_as_vault?
-          raise Exception.new("#{self.class.name} can't act as an identity vault.")
+          raise "#{self.class.name} can't act as an identity vault."
         end
         options.each do |key, value|
           if self.respond_to? "#{key}="
@@ -64,16 +69,20 @@ module RubySync::Connectors
         end
       end      
       
+
+      # Subclasses must override this. Called by perform_add to actually
+      # store the new record in the datastore. Returned value will be used
+      # as the association id if this connector is acting as the client.
+      def add id, operations
+        raise "add method not implemented"
+      end
+      
       
       # Override this to return a string that will be included within the class definition of
       # of configurations based on your connector.
       def self.sample_config
       end
       
-      # Override this to perform actions that must be performed the
-      # when the connector starts running. (Eg, opening network connections)
-      def started
-      end
       
       # Subclasses must override this to
       # interface with the external system and generate entries for every
@@ -107,31 +116,32 @@ module RubySync::Connectors
             unless self[key]
               yield RubySync::Event.delete(self, key)
               dbm.delete key
+              if is_vault? and @pipeline
+                association = association_for @pipeline.association_context, key
+                remove_association association
+              end
             end
           end
         end        
       end
       
       def digest(o)
-        Digest::MD5.hexdigest(Marshal.dump(o))
+        Digest::MD5.hexdigest(o.to_yaml)
       end
       
-      # Override this to perform actions that must be performed when
-      # the connector exits (eg closing network conections).
-      def stopped; end
 
 
       # Call each_change repeatedly (or once if in once_only mode)
       # to generate events.
       # Should generally only be called by the pipeline to which it is attached.
       def start &blk
-        log.info "#{name}: Started"
+        log.debug "#{name}: Started"
         @running = true
-        started()
+        sync_started()
         while @running
           each_change do |event|
             if event.type == :force_resync
-              each_entry &blk
+              each_entry(&blk)
               next
             end
             if is_delete_echo?(event) || is_echo?(event)
@@ -150,9 +160,26 @@ module RubySync::Connectors
             sleep 1
           end
         end
-        stopped
+        sync_stopped
       end
 
+      
+      # Called by start() after last call to each_change or each_entry
+      def sync_stopped; end
+      
+      # Called by start() before first call to each_change or each_entry
+      def sync_started; end
+
+      # Override this to perform actions that must be performed the
+      # when the connector starts running. (Eg, opening network connections)
+      def started
+      end
+      
+      # Override this to perform actions that must be performed when
+      # the connector exits (eg closing network conections).
+      def stopped; end
+
+      
       # Politely stop the connector.
       def stop
         log.info "#{name}: Attempting to stop"
@@ -198,12 +225,6 @@ module RubySync::Connectors
       # Whether this connector is capable of acting as a vault.
       # The vault is responsible for storing the association key of the client application
       # and must be able to retrieve records for that association key.
-      # Typically, databases and directories can act as vaults, text documents and HR or finance
-      # applications probably can't.
-      # To enable a connector to act as a vault, define the following methods:
-      # => path_for_foreign_key(pipeline_id, key)
-      # => foreign_key_for(path)
-      # and associate_with_foreign_key(key, path).
       def can_act_as_vault?
         defined? associate and
         defined? path_for_association and
@@ -215,11 +236,10 @@ module RubySync::Connectors
 
       # Store association for the given path
       def associate association, path
-        DBM.open(path_to_association_dbm_filename) do |dbm|
-          assocs_string = dbm[path.to_s]
-          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
-          assocs[association.context] = association.key
-          dbm[path.to_s] = Marshal.dump(assocs)
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s] || {}
+          assocs[association.context.to_s] = association.key.to_s
+          dbm[path.to_s] = assocs
         end
         DBM.open(association_to_path_dbm_filename) do |dbm|
           dbm[association.to_s] = path
@@ -233,45 +253,28 @@ module RubySync::Connectors
         end
       end
       
-      # Default implementation does nothing
       def associations_for path
-        DBM.open(path_to_association_dbm_filename) do |dbm|
-          assocs_string = dbm[path.to_s]
-          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs =  dbm[path.to_s]
           assocs.values
         end
       end
 
-      # Default implementation does nothing
+
       def remove_association association
         path = nil
         DBM.open(association_to_path_dbm_filename) do |dbm|
           return unless path =dbm.delete(association.to_s)
         end
-        DBM.open(path_to_association_dbm_filename) do |dbm|
-          assocs_string = dbm[path]
-          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
-          assocs.delete(association.context) and dbm[path.to_s] = Marshal.dump(assocs)
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s]
+          assocs.delete(association.context) and dbm[path.to_s] = assocs
         end
       end
 
-      # Could be more efficient for the default case where the
-      # associations are actually stored as a serialized hash but
-      # then it wouldn't be as generic and other implementations would
-      # have to reimplement it.
-      # def association_key_for context, path
-      #   raise "#{name} is not a vault." unless is_vault?
-      #   associations_for(path).each do |assoc|
-      #     (c, key) = assoc.split(RubySync::Association.delimiter, 2)
-      #     return key if c == context 
-      #   end
-      #   return nil
-      # end
-
       def association_key_for context, path
-        DBM.open(path_to_association_dbm_filename) do |dbm|
-          assocs_string = dbm[path.to_s]
-          assocs = (assocs_string)? Marshal.load(assocs_string) : {}
+        YAML::DBM.open(path_to_association_dbm_filename) do |dbm|
+          assocs = dbm[path.to_s] || {}
           assocs[context.to_s]
         end
       end
@@ -356,40 +359,6 @@ module RubySync::Connectors
       end
 
 
-      # Performs the given operations on the given record. The record is a
-      # Hash in which each key is a field name and each value is an array of
-      # values for that field.
-      # Operations is an Array of RubySync::Operation objects to be performed on the record.
-      def perform_operations operations, record={}
-        operations.each do |op|
-          unless op.instance_of? RubySync::Operation
-            log.warn "!!!!!!!!!!  PROBLEM, DUMP FOLLOWS: !!!!!!!!!!!!!!"
-            p op
-          end
-          case op.type
-          when :add
-            if record[op.subject]
-              existing = record[op.subject].as_array
-              (existing & op.values).empty? or
-                raise Exception.new("Attempt to add duplicate elements to #{name}")
-              record[op.subject] =  existing + op.values
-            else
-              record[op.subject] = op.values
-            end
-          when :replace
-            record[op.subject] = op.values
-          when :delete
-            if value == nil || value == "" || value == []
-              record.delete(op.subject)
-            else
-              record[op.subject] -= values
-            end
-          else
-            raise Exception.new("Unknown operation '#{op.type}'")
-          end
-        end
-        return record
-      end
 
 
       # Return an array of possible fields for this connector.