rubysync 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. data/HISTORY.txt +4 -0
  2. data/Manifest.txt +25 -12
  3. data/README.txt +0 -2
  4. data/bin/rubysync +20 -6
  5. data/bin/rubysync.rb +333 -0
  6. data/docs/in_pipeline.graffle +2690 -0
  7. data/docs/init_openldap.ldif +11 -0
  8. data/docs/out_pipeline.graffle +3274 -0
  9. data/docs/schema/99rubysync.ldif +27 -0
  10. data/docs/schema/rubysync.schema +16 -0
  11. data/docs/to_sync.txt +15 -0
  12. data/docs/walkthru.txt +186 -0
  13. data/lib/ruby_sync.rb +7 -29
  14. data/lib/ruby_sync/connectors/base_connector.rb +55 -86
  15. data/lib/ruby_sync/connectors/csv_file_connector.rb +16 -4
  16. data/lib/ruby_sync/connectors/ldap_associations.rb +126 -0
  17. data/lib/ruby_sync/connectors/ldap_changelog_connector.rb +127 -0
  18. data/lib/ruby_sync/connectors/ldap_connector.rb +29 -192
  19. data/lib/ruby_sync/connectors/memory_connector.rb +1 -1
  20. data/lib/ruby_sync/connectors/xml_connector.rb +105 -32
  21. data/lib/ruby_sync/event.rb +40 -12
  22. data/lib/ruby_sync/operation.rb +18 -2
  23. data/lib/ruby_sync/pipelines/base_pipeline.rb +44 -6
  24. data/lib/ruby_sync/util/utilities.rb +97 -4
  25. data/lib/rubysync.rb +1 -1
  26. data/rubysync.tmproj +279 -59
  27. data/test/.LCKts_rubysync.rb~ +1 -0
  28. data/test/ruby_sync_test.rb +9 -4
  29. data/test/{test_active_record_vault.rb → tc_active_record_connector.rb} +11 -7
  30. data/test/{test_base_connector.rb → tc_base_connector.rb} +1 -1
  31. data/test/{test_base_pipeline.rb → tc_base_pipeline.rb} +1 -1
  32. data/test/tc_changelog_ldap_connector.rb +93 -0
  33. data/test/{test_csv_file_connector.rb → tc_csv_file_connector.rb} +14 -5
  34. data/test/{test_event.rb → tc_event.rb} +1 -1
  35. data/test/{test_ldap_changelog.rb → tc_ldap_changelog.rb} +1 -1
  36. data/test/{test_ldap_connector.rb → tc_ldap_connector.rb} +20 -22
  37. data/test/{test_ldap_vault.rb → tc_ldap_vault.rb} +2 -2
  38. data/test/{test_ldif.rb → tc_ldif.rb} +1 -1
  39. data/test/{test_memory_connectors.rb → tc_memory_connectors.rb} +10 -6
  40. data/test/{test_rubysync.rb → tc_rubysync.rb} +4 -4
  41. data/test/tc_transformation.rb +71 -0
  42. data/test/{test_utilities.rb → tc_utilities.rb} +28 -1
  43. data/test/tc_xml_connectors.rb +107 -6
  44. data/test/ts_rubysync.rb +11 -6
  45. metadata +33 -28
@@ -15,7 +15,7 @@
15
15
 
16
16
 
17
17
  require "yaml"
18
-
18
+ defined?(RubySync::Connectors::BaseConnector) or require 'ruby_sync/connectors/base_connector'
19
19
 
20
20
  module RubySync::Connectors
21
21
  class MemoryConnector < RubySync::Connectors::BaseConnector
@@ -21,78 +21,151 @@ require 'ruby_sync'
21
21
  $VERBOSE = false
22
22
  #require 'xmlsimple'
23
23
  #$VERBOSE = true
24
+ require 'rexml/document'
24
25
 
25
-
26
+ class REXML::Document
27
+
28
+ def entry_element_for id
29
+ # root.each_element_with_attribute('id', id) do |element|
30
+ root.each_element("entry[@id='#{id}']") do |element|
31
+ return element
32
+ end
33
+ nil
34
+ end
35
+
36
+ end
26
37
 
27
38
  module RubySync::Connectors
28
39
  class XmlConnector < RubySync::Connectors::BaseConnector
40
+
41
+ include REXML
29
42
 
30
43
  option :filename
31
44
 
32
45
  def each_entry
33
- with_xml(:read_only=>true) do |content|
34
- content.each do |entry|
35
- yield entry[0], entry[1][0]
46
+ with_xml(:read_only=>true) do |xml|
47
+ xml.root.each_element("entry") do |element|
48
+ yield element.attribute('id').value, to_entry(element)
36
49
  end
37
50
  end
38
51
  end
39
52
 
53
+
40
54
  def add id, operations
41
- with_xml do |content|
42
- content[id] = perform_operations(operations)
55
+ entry = nil
56
+ with_xml do |xml|
57
+ xml.entry_element_for(id) and raise "Element '#{id}' already exists."
58
+ entry = perform_operations(operations)
43
59
  end
60
+ self[id] = entry
44
61
  id
45
62
  end
46
63
 
47
64
  def modify id, operations
48
- with_xml do |content|
49
- existing = content[id] && content[id][0] || {}
50
- content[id] = perform_operations(operations, existing)
65
+ entry = nil
66
+ with_xml do |xml|
67
+ existing_entry = to_entry(xml.entry_element_for(id))
68
+ entry = perform_operations(operations, existing_entry)
51
69
  end
70
+ self[id] = entry
52
71
  id
53
72
  end
54
73
 
55
74
  def delete id
56
- with_xml do |content|
57
- content.delete(id)
75
+ xpath = "//entry[@id='#{id}']"
76
+ with_xml do |xml|
77
+ xml.root.delete_element xpath
58
78
  end
59
- id
60
79
  end
61
80
 
62
81
  def [](id)
63
- value = nil
64
- with_xml(:read_only=>true) do |content|
65
- value = content[id] && content[id][0]
82
+ with_xml(:read_only=>true) do |xml|
83
+ element = xml.entry_element_for(id)
84
+ return (element)? to_entry(element) : nil
66
85
  end
67
- value
68
86
  end
87
+
88
+
69
89
 
70
90
  def []=(id, value)
71
- with_xml do |content|
72
- content[id] = value
91
+ with_xml do |xml|
92
+ new_child = to_xml(id, value)
93
+ if old_child = xml.entry_element_for(id)
94
+ xml.root.replace_child(old_child, new_child)
95
+ else
96
+ xml.root << new_child
97
+ end
98
+ end
99
+ end
100
+
101
+
102
+ def to_xml key, entry
103
+ el = Element.new("entry")
104
+ el.add_attribute('id', key)
105
+ entry.each do |key, values|
106
+ el << attr = Element.new("attr")
107
+ attr.add_attribute 'name', key
108
+ as_array(values).each do |value|
109
+ value_el = Element.new('value')
110
+ attr << value_el.add_text(value)
111
+ end
73
112
  end
113
+ el
74
114
  end
115
+
116
+
117
+
75
118
 
119
+ def to_entry entry_element
120
+ entry = {}
121
+ if entry_element
122
+ entry_element.each_element("attr") do |child|
123
+ entry[child.attribute('name').value] = values = []
124
+ child.each_element("value") do |value_element|
125
+ values << value_element.text
126
+ end
127
+ end
128
+ end
129
+ entry
130
+ end
131
+
76
132
 
77
133
  def self.sample_config
78
- return <<END
79
- #
80
- # "filename" should be the full name of the file containing
81
- # the xml representation of the synchronized content.
82
- # You probably want to change this:
83
- #
84
- filename "/tmp/rubysync.xml"
85
- END
134
+ return %q(
135
+ #
136
+ # "filename" should be the full name of the file containing
137
+ # the xml representation of the synchronized content.
138
+ # You probably want to change this:
139
+ #
140
+ filename "/tmp/rubysync.xml"
141
+ )
86
142
  end
87
143
 
88
- private
89
144
 
90
-
145
+ # Should be re-entrant within a single thread but isn't
146
+ # thread-safe.
91
147
  def with_xml options={}
92
- content = (File.exist?(filename))? content = XmlSimple.xml_in(filename) : {}
93
- yield content
94
- XmlSimple.xml_out(content, {'OutputFile'=>filename}) unless options[:read_only]
148
+ unless @with_xml_invoked
149
+ begin
150
+ @with_xml_invoked = true
151
+ File.exist?(filename) or File.open(filename,'w') {|file| file.write('<entries/>')}
152
+ File.open(filename, "r") do |file|
153
+ file.flock(File::LOCK_EX)
154
+ @xml = Document.new(file)
155
+ begin
156
+ yield @xml
157
+ ensure
158
+ File.open(filename, "w") do |out|
159
+ @xml.write out
160
+ end
161
+ end
162
+ end
163
+ ensure
164
+ @with_xml_invoked = false
165
+ end
166
+ else # this is a nested call so we don't need to read or write the file
167
+ yield @xml
168
+ end
95
169
  end
96
-
97
170
  end
98
171
  end
@@ -43,6 +43,8 @@ module RubySync
43
43
  # array of RubySync::Operations describing changes to the attributes of the
44
44
  # record.
45
45
  class Event
46
+
47
+ include RubySync::Utilities
46
48
 
47
49
  attr_accessor :type, # delete, add, modify ...
48
50
  :source,
@@ -93,9 +95,12 @@ module RubySync
93
95
  self.association && self.association.context && self.association.key
94
96
  end
95
97
 
98
+ # Reduces the operations in this event to those that will
99
+ # alter the target record
96
100
  def merge other
97
- # TODO implement merge
98
- log.warn "Event.merge not yet implemented"
101
+ # other.type == :add or raise "Can only merge with add events"
102
+ # record = perform_operations(other.payload)
103
+ payload = effective_operations(@payload, other)
99
104
  end
100
105
 
101
106
  # Retrieves all known values for the record affected by this event and
@@ -133,7 +138,7 @@ module RubySync
133
138
  def sets_value? subject, value=nil
134
139
  return false if @payload == nil
135
140
  @payload.reverse_each do |op|
136
- return true if op.subject == subject.to_s && (value == nil || op.values == value.as_array)
141
+ return true if op.subject == subject.to_s && (value == nil || op.values == as_array(value))
137
142
  end
138
143
  return false
139
144
  end
@@ -141,31 +146,40 @@ module RubySync
141
146
  # Remove any operations from the payload that affect fields with the given key or
142
147
  # keys (key can be a single field name or an array of field names).
143
148
  def drop_changes_to subject
144
- subjects = subject.as_array
149
+ subjects = as_array(subject).map {|s| s.to_s}
145
150
  uncommitted_operations
146
151
  @uncommitted_operations = @uncommitted_operations.delete_if {|op| subjects.include? op.subject }
147
152
  end
148
153
 
149
154
  def drop_all_but_changes_to subject
150
- subjects = subject.as_array.map {|s| s.to_s}
155
+ subjects = as_array(subject).map {|s| s.to_s}
151
156
  @uncommitted_operations = uncommitted_operations.delete_if {|op| !subjects.include?(op.subject.to_s)}
152
157
  end
153
158
 
154
159
  # Add a value to a given subject unless it already sets a value
155
160
  def add_default field_name, value
156
- add_value field_name, value unless sets_value? field_name
161
+ add_value(field_name.to_s, value) unless sets_value? field_name.to_s
157
162
  end
158
163
 
159
164
 
160
165
  def add_value field_name, value
161
- uncommitted_operations << Operation.new(:add, field_name, value.as_array)
166
+ uncommitted_operations << Operation.new(:add, field_name.to_s, as_array(value))
162
167
  end
163
168
 
164
169
  def set_value field_name, value
165
- uncommitted_operations << Operation.new(:replace, field_name, value.as_array)
170
+ uncommitted_operations << Operation.new(:replace, field_name.to_s, as_array(value))
166
171
  end
167
172
 
168
-
173
+ def values_for field_name
174
+ values = perform_operations @payload, {}, :subjects=>[field_name.to_s]
175
+ values[field_name.to_s]
176
+ end
177
+
178
+ def value_for field_name
179
+ values = values_for field_name
180
+ (values)? values[0] : nil
181
+ end
182
+
169
183
  def uncommitted_operations
170
184
  @uncommitted_operations ||= @payload || []
171
185
  return @uncommitted_operations
@@ -181,7 +195,7 @@ module RubySync
181
195
  # first.
182
196
  def append new_operations
183
197
  uncommitted_operations
184
- @uncommitted_operations += new_operations.as_array
198
+ @uncommitted_operations += as_array(new_operations)
185
199
  end
186
200
 
187
201
  # Rollback any changes that
@@ -195,6 +209,20 @@ module RubySync
195
209
  @uncommitted_operations = nil
196
210
  end
197
211
  end
212
+
213
+ # Typically this will be called in the 'transform_in' and 'transform_out'
214
+ # blocks in a pipeline configuration.
215
+ def map(left, right=nil, &blk)
216
+ if right
217
+ drop_changes_to left
218
+ @uncommitted_operations = uncommitted_operations.map do |op|
219
+ (op.subject.to_s == right.to_s)? op.same_but_on(left.to_s) : op
220
+ end
221
+ elsif blk and [:add, :modify].include? @type
222
+ drop_changes_to left.to_s
223
+ uncommitted_operations << RubySync::Operation.replace(left.to_s, blk.call)
224
+ end
225
+ end
198
226
 
199
227
  protected
200
228
 
@@ -217,7 +245,7 @@ module RubySync
217
245
  # the specified subject
218
246
  def each_operation_on subject
219
247
  return unless payload
220
- subjects = subject.as_array.map {|s| s.to_s}
248
+ subjects = as_array(subject).map {|s| s.to_s}
221
249
  payload.each do |op|
222
250
  if subjects.include?(op.subject.to_s)
223
251
  yield(op)
@@ -231,4 +259,4 @@ module RubySync
231
259
  end
232
260
 
233
261
 
234
-
262
+
@@ -18,6 +18,8 @@ module RubySync
18
18
  # Operations that may be performed on an attribute
19
19
  class Operation
20
20
 
21
+ include RubySync::Utilities
22
+
21
23
  attr_accessor :type, :subject, :values
22
24
 
23
25
 
@@ -40,6 +42,12 @@ module RubySync
40
42
  self.values = values
41
43
  end
42
44
 
45
+ def ==(o)
46
+ subject == o.subject &&
47
+ type == o.type &&
48
+ values == o.values
49
+ end
50
+
43
51
  remove_method :type=
44
52
  def type=(type)
45
53
  unless [:add, :delete, :replace].include? type.to_sym
@@ -50,9 +58,17 @@ module RubySync
50
58
 
51
59
  remove_method :values=
52
60
  def values=(values)
53
- @values = values.as_array
61
+ @values = as_array(values)
54
62
  end
55
63
 
64
+ def value
65
+ @values[0]
66
+ end
67
+
68
+ def value= new_value
69
+ @values = new_value.as_array
70
+ end
71
+
56
72
  # Returns a duplicate of this operation but with the subject
57
73
  # changed to the specified subject
58
74
  def same_but_on subject
@@ -79,4 +95,4 @@ module RubySync
79
95
 
80
96
 
81
97
  end
82
- end
98
+ end
@@ -40,7 +40,10 @@ module RubySync
40
40
 
41
41
  include RubySync::Utilities
42
42
 
43
+ attr_accessor :delay # delay in seconds between checking connectors
44
+
43
45
  def initialize
46
+ @delay = 5
44
47
  end
45
48
 
46
49
  def name
@@ -61,7 +64,11 @@ module RubySync
61
64
  options[:name] ||= "#{self.name}(vault)"
62
65
  options[:is_vault] = true
63
66
  class_def 'vault' do
64
- @vault ||= eval("::" + class_name).new(options)
67
+ unless @vault
68
+ @vault = eval("::" + class_name).new(options)
69
+ @vault.pipeline = self
70
+ end
71
+ @vault
65
72
  end
66
73
  end
67
74
 
@@ -177,7 +184,7 @@ module RubySync
177
184
  # Override to implement some kind of matching
178
185
  def out_match event
179
186
  log.debug "Default matching rule - source path exists on client?"
180
- client.respond_to?(:'[]') and client[event.source_path]
187
+ client.respond_to?('[]') and client[event.source_path]
181
188
  false
182
189
  end
183
190
 
@@ -216,27 +223,58 @@ module RubySync
216
223
  # end
217
224
 
218
225
  # Execute the pipeline once then return.
219
- # TODO Consider making this run in and out simultaneously
220
226
  def run_once
221
227
  log.info "Running #{name} pipeline once"
228
+ started
222
229
  run_in_once
223
230
  run_out_once
231
+ stopped
232
+ end
233
+
234
+ def started
235
+ client.started
236
+ vault.started
237
+ end
238
+
239
+ def stopped
240
+ client.stopped
241
+ vault.stopped
224
242
  end
225
243
 
226
244
  # Execute the in pipe once and then return
227
245
  def run_in_once
228
- log.info "Running #{name} 'in' pipeline once"
246
+ log.debug "Running #{name} 'in' pipeline once"
229
247
  client.once_only = true
230
248
  client.start {|event| in_handler(event)}
231
249
  end
232
250
 
233
251
  # Execute the out pipe once and then return
234
252
  def run_out_once
235
- log.info "Running #{name} 'out' pipeline once"
253
+ log.debug "Running #{name} 'out' pipeline once"
236
254
  vault.once_only = true
237
255
  vault.start {|event| out_handler(event)}
238
256
  end
239
257
 
258
+ def start
259
+ log.info "Starting #{name} pipeline"
260
+ @running = true
261
+ trap("SIGINT") {self.stop}
262
+ started
263
+ while @running
264
+ run_in_once
265
+ run_out_once
266
+ sleep delay
267
+ end
268
+ stopped
269
+ log.info "#{name} stopped."
270
+ end
271
+
272
+ def stop
273
+ log.info "#{name} stopping..."
274
+ @running = false
275
+ Thread.main.run # i thought this would wake the thread from its sleep
276
+ # but it seems to have no effect.
277
+ end
240
278
 
241
279
  # Override to process the event generated by the publisher before any other processing is done.
242
280
  # Return false to veto the event.
@@ -287,7 +325,7 @@ module RubySync
287
325
 
288
326
  def in_match event
289
327
  log.debug "Default match rule - source path exists in vault"
290
- vault.respond_to?(:'[]') and vault[event.source_path]
328
+ vault.respond_to?('[]') and vault[event.source_path]
291
329
  end
292
330
 
293
331
  # If client_to_vault_map is defined (usually by map_client_to_vault)