rubysync 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.txt +4 -0
- data/Manifest.txt +25 -12
- data/README.txt +0 -2
- data/bin/rubysync +20 -6
- data/bin/rubysync.rb +333 -0
- data/docs/in_pipeline.graffle +2690 -0
- data/docs/init_openldap.ldif +11 -0
- data/docs/out_pipeline.graffle +3274 -0
- data/docs/schema/99rubysync.ldif +27 -0
- data/docs/schema/rubysync.schema +16 -0
- data/docs/to_sync.txt +15 -0
- data/docs/walkthru.txt +186 -0
- data/lib/ruby_sync.rb +7 -29
- data/lib/ruby_sync/connectors/base_connector.rb +55 -86
- data/lib/ruby_sync/connectors/csv_file_connector.rb +16 -4
- data/lib/ruby_sync/connectors/ldap_associations.rb +126 -0
- data/lib/ruby_sync/connectors/ldap_changelog_connector.rb +127 -0
- data/lib/ruby_sync/connectors/ldap_connector.rb +29 -192
- data/lib/ruby_sync/connectors/memory_connector.rb +1 -1
- data/lib/ruby_sync/connectors/xml_connector.rb +105 -32
- data/lib/ruby_sync/event.rb +40 -12
- data/lib/ruby_sync/operation.rb +18 -2
- data/lib/ruby_sync/pipelines/base_pipeline.rb +44 -6
- data/lib/ruby_sync/util/utilities.rb +97 -4
- data/lib/rubysync.rb +1 -1
- data/rubysync.tmproj +279 -59
- data/test/.LCKts_rubysync.rb~ +1 -0
- data/test/ruby_sync_test.rb +9 -4
- data/test/{test_active_record_vault.rb → tc_active_record_connector.rb} +11 -7
- data/test/{test_base_connector.rb → tc_base_connector.rb} +1 -1
- data/test/{test_base_pipeline.rb → tc_base_pipeline.rb} +1 -1
- data/test/tc_changelog_ldap_connector.rb +93 -0
- data/test/{test_csv_file_connector.rb → tc_csv_file_connector.rb} +14 -5
- data/test/{test_event.rb → tc_event.rb} +1 -1
- data/test/{test_ldap_changelog.rb → tc_ldap_changelog.rb} +1 -1
- data/test/{test_ldap_connector.rb → tc_ldap_connector.rb} +20 -22
- data/test/{test_ldap_vault.rb → tc_ldap_vault.rb} +2 -2
- data/test/{test_ldif.rb → tc_ldif.rb} +1 -1
- data/test/{test_memory_connectors.rb → tc_memory_connectors.rb} +10 -6
- data/test/{test_rubysync.rb → tc_rubysync.rb} +4 -4
- data/test/tc_transformation.rb +71 -0
- data/test/{test_utilities.rb → tc_utilities.rb} +28 -1
- data/test/tc_xml_connectors.rb +107 -6
- data/test/ts_rubysync.rb +11 -6
- metadata +33 -28
@@ -21,78 +21,151 @@ require 'ruby_sync'
|
|
21
21
|
$VERBOSE = false
|
22
22
|
#require 'xmlsimple'
|
23
23
|
#$VERBOSE = true
|
24
|
+
require 'rexml/document'
|
24
25
|
|
25
|
-
|
26
|
+
class REXML::Document
|
27
|
+
|
28
|
+
def entry_element_for id
|
29
|
+
# root.each_element_with_attribute('id', id) do |element|
|
30
|
+
root.each_element("entry[@id='#{id}']") do |element|
|
31
|
+
return element
|
32
|
+
end
|
33
|
+
nil
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
26
37
|
|
27
38
|
module RubySync::Connectors
|
28
39
|
class XmlConnector < RubySync::Connectors::BaseConnector
|
40
|
+
|
41
|
+
include REXML
|
29
42
|
|
30
43
|
option :filename
|
31
44
|
|
32
45
|
def each_entry
|
33
|
-
with_xml(:read_only=>true) do |
|
34
|
-
|
35
|
-
yield
|
46
|
+
with_xml(:read_only=>true) do |xml|
|
47
|
+
xml.root.each_element("entry") do |element|
|
48
|
+
yield element.attribute('id').value, to_entry(element)
|
36
49
|
end
|
37
50
|
end
|
38
51
|
end
|
39
52
|
|
53
|
+
|
40
54
|
def add id, operations
|
41
|
-
|
42
|
-
|
55
|
+
entry = nil
|
56
|
+
with_xml do |xml|
|
57
|
+
xml.entry_element_for(id) and raise "Element '#{id}' already exists."
|
58
|
+
entry = perform_operations(operations)
|
43
59
|
end
|
60
|
+
self[id] = entry
|
44
61
|
id
|
45
62
|
end
|
46
63
|
|
47
64
|
def modify id, operations
|
48
|
-
|
49
|
-
|
50
|
-
|
65
|
+
entry = nil
|
66
|
+
with_xml do |xml|
|
67
|
+
existing_entry = to_entry(xml.entry_element_for(id))
|
68
|
+
entry = perform_operations(operations, existing_entry)
|
51
69
|
end
|
70
|
+
self[id] = entry
|
52
71
|
id
|
53
72
|
end
|
54
73
|
|
55
74
|
def delete id
|
56
|
-
|
57
|
-
|
75
|
+
xpath = "//entry[@id='#{id}']"
|
76
|
+
with_xml do |xml|
|
77
|
+
xml.root.delete_element xpath
|
58
78
|
end
|
59
|
-
id
|
60
79
|
end
|
61
80
|
|
62
81
|
def [](id)
|
63
|
-
|
64
|
-
|
65
|
-
|
82
|
+
with_xml(:read_only=>true) do |xml|
|
83
|
+
element = xml.entry_element_for(id)
|
84
|
+
return (element)? to_entry(element) : nil
|
66
85
|
end
|
67
|
-
value
|
68
86
|
end
|
87
|
+
|
88
|
+
|
69
89
|
|
70
90
|
def []=(id, value)
|
71
|
-
with_xml do |
|
72
|
-
|
91
|
+
with_xml do |xml|
|
92
|
+
new_child = to_xml(id, value)
|
93
|
+
if old_child = xml.entry_element_for(id)
|
94
|
+
xml.root.replace_child(old_child, new_child)
|
95
|
+
else
|
96
|
+
xml.root << new_child
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
|
102
|
+
def to_xml key, entry
|
103
|
+
el = Element.new("entry")
|
104
|
+
el.add_attribute('id', key)
|
105
|
+
entry.each do |key, values|
|
106
|
+
el << attr = Element.new("attr")
|
107
|
+
attr.add_attribute 'name', key
|
108
|
+
as_array(values).each do |value|
|
109
|
+
value_el = Element.new('value')
|
110
|
+
attr << value_el.add_text(value)
|
111
|
+
end
|
73
112
|
end
|
113
|
+
el
|
74
114
|
end
|
115
|
+
|
116
|
+
|
117
|
+
|
75
118
|
|
119
|
+
def to_entry entry_element
|
120
|
+
entry = {}
|
121
|
+
if entry_element
|
122
|
+
entry_element.each_element("attr") do |child|
|
123
|
+
entry[child.attribute('name').value] = values = []
|
124
|
+
child.each_element("value") do |value_element|
|
125
|
+
values << value_element.text
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
129
|
+
entry
|
130
|
+
end
|
131
|
+
|
76
132
|
|
77
133
|
def self.sample_config
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
134
|
+
return %q(
|
135
|
+
#
|
136
|
+
# "filename" should be the full name of the file containing
|
137
|
+
# the xml representation of the synchronized content.
|
138
|
+
# You probably want to change this:
|
139
|
+
#
|
140
|
+
filename "/tmp/rubysync.xml"
|
141
|
+
)
|
86
142
|
end
|
87
143
|
|
88
|
-
private
|
89
144
|
|
90
|
-
|
145
|
+
# Should be re-entrant within a single thread but isn't
|
146
|
+
# thread-safe.
|
91
147
|
def with_xml options={}
|
92
|
-
|
93
|
-
|
94
|
-
|
148
|
+
unless @with_xml_invoked
|
149
|
+
begin
|
150
|
+
@with_xml_invoked = true
|
151
|
+
File.exist?(filename) or File.open(filename,'w') {|file| file.write('<entries/>')}
|
152
|
+
File.open(filename, "r") do |file|
|
153
|
+
file.flock(File::LOCK_EX)
|
154
|
+
@xml = Document.new(file)
|
155
|
+
begin
|
156
|
+
yield @xml
|
157
|
+
ensure
|
158
|
+
File.open(filename, "w") do |out|
|
159
|
+
@xml.write out
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
ensure
|
164
|
+
@with_xml_invoked = false
|
165
|
+
end
|
166
|
+
else # this is a nested call so we don't need to read or write the file
|
167
|
+
yield @xml
|
168
|
+
end
|
95
169
|
end
|
96
|
-
|
97
170
|
end
|
98
171
|
end
|
data/lib/ruby_sync/event.rb
CHANGED
@@ -43,6 +43,8 @@ module RubySync
|
|
43
43
|
# array of RubySync::Operations describing changes to the attributes of the
|
44
44
|
# record.
|
45
45
|
class Event
|
46
|
+
|
47
|
+
include RubySync::Utilities
|
46
48
|
|
47
49
|
attr_accessor :type, # delete, add, modify ...
|
48
50
|
:source,
|
@@ -93,9 +95,12 @@ module RubySync
|
|
93
95
|
self.association && self.association.context && self.association.key
|
94
96
|
end
|
95
97
|
|
98
|
+
# Reduces the operations in this event to those that will
|
99
|
+
# alter the target record
|
96
100
|
def merge other
|
97
|
-
|
98
|
-
|
101
|
+
# other.type == :add or raise "Can only merge with add events"
|
102
|
+
# record = perform_operations(other.payload)
|
103
|
+
payload = effective_operations(@payload, other)
|
99
104
|
end
|
100
105
|
|
101
106
|
# Retrieves all known values for the record affected by this event and
|
@@ -133,7 +138,7 @@ module RubySync
|
|
133
138
|
def sets_value? subject, value=nil
|
134
139
|
return false if @payload == nil
|
135
140
|
@payload.reverse_each do |op|
|
136
|
-
return true if op.subject == subject.to_s && (value == nil || op.values == value
|
141
|
+
return true if op.subject == subject.to_s && (value == nil || op.values == as_array(value))
|
137
142
|
end
|
138
143
|
return false
|
139
144
|
end
|
@@ -141,31 +146,40 @@ module RubySync
|
|
141
146
|
# Remove any operations from the payload that affect fields with the given key or
|
142
147
|
# keys (key can be a single field name or an array of field names).
|
143
148
|
def drop_changes_to subject
|
144
|
-
subjects = subject.
|
149
|
+
subjects = as_array(subject).map {|s| s.to_s}
|
145
150
|
uncommitted_operations
|
146
151
|
@uncommitted_operations = @uncommitted_operations.delete_if {|op| subjects.include? op.subject }
|
147
152
|
end
|
148
153
|
|
149
154
|
def drop_all_but_changes_to subject
|
150
|
-
subjects = subject.
|
155
|
+
subjects = as_array(subject).map {|s| s.to_s}
|
151
156
|
@uncommitted_operations = uncommitted_operations.delete_if {|op| !subjects.include?(op.subject.to_s)}
|
152
157
|
end
|
153
158
|
|
154
159
|
# Add a value to a given subject unless it already sets a value
|
155
160
|
def add_default field_name, value
|
156
|
-
add_value
|
161
|
+
add_value(field_name.to_s, value) unless sets_value? field_name.to_s
|
157
162
|
end
|
158
163
|
|
159
164
|
|
160
165
|
def add_value field_name, value
|
161
|
-
uncommitted_operations << Operation.new(:add, field_name, value
|
166
|
+
uncommitted_operations << Operation.new(:add, field_name.to_s, as_array(value))
|
162
167
|
end
|
163
168
|
|
164
169
|
def set_value field_name, value
|
165
|
-
uncommitted_operations << Operation.new(:replace, field_name, value
|
170
|
+
uncommitted_operations << Operation.new(:replace, field_name.to_s, as_array(value))
|
166
171
|
end
|
167
172
|
|
168
|
-
|
173
|
+
def values_for field_name
|
174
|
+
values = perform_operations @payload, {}, :subjects=>[field_name.to_s]
|
175
|
+
values[field_name.to_s]
|
176
|
+
end
|
177
|
+
|
178
|
+
def value_for field_name
|
179
|
+
values = values_for field_name
|
180
|
+
(values)? values[0] : nil
|
181
|
+
end
|
182
|
+
|
169
183
|
def uncommitted_operations
|
170
184
|
@uncommitted_operations ||= @payload || []
|
171
185
|
return @uncommitted_operations
|
@@ -181,7 +195,7 @@ module RubySync
|
|
181
195
|
# first.
|
182
196
|
def append new_operations
|
183
197
|
uncommitted_operations
|
184
|
-
@uncommitted_operations += new_operations
|
198
|
+
@uncommitted_operations += as_array(new_operations)
|
185
199
|
end
|
186
200
|
|
187
201
|
# Rollback any changes that
|
@@ -195,6 +209,20 @@ module RubySync
|
|
195
209
|
@uncommitted_operations = nil
|
196
210
|
end
|
197
211
|
end
|
212
|
+
|
213
|
+
# Typically this will be called in the 'transform_in' and 'transform_out'
|
214
|
+
# blocks in a pipeline configuration.
|
215
|
+
def map(left, right=nil, &blk)
|
216
|
+
if right
|
217
|
+
drop_changes_to left
|
218
|
+
@uncommitted_operations = uncommitted_operations.map do |op|
|
219
|
+
(op.subject.to_s == right.to_s)? op.same_but_on(left.to_s) : op
|
220
|
+
end
|
221
|
+
elsif blk and [:add, :modify].include? @type
|
222
|
+
drop_changes_to left.to_s
|
223
|
+
uncommitted_operations << RubySync::Operation.replace(left.to_s, blk.call)
|
224
|
+
end
|
225
|
+
end
|
198
226
|
|
199
227
|
protected
|
200
228
|
|
@@ -217,7 +245,7 @@ module RubySync
|
|
217
245
|
# the specified subject
|
218
246
|
def each_operation_on subject
|
219
247
|
return unless payload
|
220
|
-
subjects = subject.
|
248
|
+
subjects = as_array(subject).map {|s| s.to_s}
|
221
249
|
payload.each do |op|
|
222
250
|
if subjects.include?(op.subject.to_s)
|
223
251
|
yield(op)
|
@@ -231,4 +259,4 @@ module RubySync
|
|
231
259
|
end
|
232
260
|
|
233
261
|
|
234
|
-
|
262
|
+
|
data/lib/ruby_sync/operation.rb
CHANGED
@@ -18,6 +18,8 @@ module RubySync
|
|
18
18
|
# Operations that may be performed on an attribute
|
19
19
|
class Operation
|
20
20
|
|
21
|
+
include RubySync::Utilities
|
22
|
+
|
21
23
|
attr_accessor :type, :subject, :values
|
22
24
|
|
23
25
|
|
@@ -40,6 +42,12 @@ module RubySync
|
|
40
42
|
self.values = values
|
41
43
|
end
|
42
44
|
|
45
|
+
def ==(o)
|
46
|
+
subject == o.subject &&
|
47
|
+
type == o.type &&
|
48
|
+
values == o.values
|
49
|
+
end
|
50
|
+
|
43
51
|
remove_method :type=
|
44
52
|
def type=(type)
|
45
53
|
unless [:add, :delete, :replace].include? type.to_sym
|
@@ -50,9 +58,17 @@ module RubySync
|
|
50
58
|
|
51
59
|
remove_method :values=
|
52
60
|
def values=(values)
|
53
|
-
@values = values
|
61
|
+
@values = as_array(values)
|
54
62
|
end
|
55
63
|
|
64
|
+
def value
|
65
|
+
@values[0]
|
66
|
+
end
|
67
|
+
|
68
|
+
def value= new_value
|
69
|
+
@values = new_value.as_array
|
70
|
+
end
|
71
|
+
|
56
72
|
# Returns a duplicate of this operation but with the subject
|
57
73
|
# changed to the specified subject
|
58
74
|
def same_but_on subject
|
@@ -79,4 +95,4 @@ module RubySync
|
|
79
95
|
|
80
96
|
|
81
97
|
end
|
82
|
-
end
|
98
|
+
end
|
@@ -40,7 +40,10 @@ module RubySync
|
|
40
40
|
|
41
41
|
include RubySync::Utilities
|
42
42
|
|
43
|
+
attr_accessor :delay # delay in seconds between checking connectors
|
44
|
+
|
43
45
|
def initialize
|
46
|
+
@delay = 5
|
44
47
|
end
|
45
48
|
|
46
49
|
def name
|
@@ -61,7 +64,11 @@ module RubySync
|
|
61
64
|
options[:name] ||= "#{self.name}(vault)"
|
62
65
|
options[:is_vault] = true
|
63
66
|
class_def 'vault' do
|
64
|
-
@vault
|
67
|
+
unless @vault
|
68
|
+
@vault = eval("::" + class_name).new(options)
|
69
|
+
@vault.pipeline = self
|
70
|
+
end
|
71
|
+
@vault
|
65
72
|
end
|
66
73
|
end
|
67
74
|
|
@@ -177,7 +184,7 @@ module RubySync
|
|
177
184
|
# Override to implement some kind of matching
|
178
185
|
def out_match event
|
179
186
|
log.debug "Default matching rule - source path exists on client?"
|
180
|
-
client.respond_to?(
|
187
|
+
client.respond_to?('[]') and client[event.source_path]
|
181
188
|
false
|
182
189
|
end
|
183
190
|
|
@@ -216,27 +223,58 @@ module RubySync
|
|
216
223
|
# end
|
217
224
|
|
218
225
|
# Execute the pipeline once then return.
|
219
|
-
# TODO Consider making this run in and out simultaneously
|
220
226
|
def run_once
|
221
227
|
log.info "Running #{name} pipeline once"
|
228
|
+
started
|
222
229
|
run_in_once
|
223
230
|
run_out_once
|
231
|
+
stopped
|
232
|
+
end
|
233
|
+
|
234
|
+
def started
|
235
|
+
client.started
|
236
|
+
vault.started
|
237
|
+
end
|
238
|
+
|
239
|
+
def stopped
|
240
|
+
client.stopped
|
241
|
+
vault.stopped
|
224
242
|
end
|
225
243
|
|
226
244
|
# Execute the in pipe once and then return
|
227
245
|
def run_in_once
|
228
|
-
log.
|
246
|
+
log.debug "Running #{name} 'in' pipeline once"
|
229
247
|
client.once_only = true
|
230
248
|
client.start {|event| in_handler(event)}
|
231
249
|
end
|
232
250
|
|
233
251
|
# Execute the out pipe once and then return
|
234
252
|
def run_out_once
|
235
|
-
log.
|
253
|
+
log.debug "Running #{name} 'out' pipeline once"
|
236
254
|
vault.once_only = true
|
237
255
|
vault.start {|event| out_handler(event)}
|
238
256
|
end
|
239
257
|
|
258
|
+
def start
|
259
|
+
log.info "Starting #{name} pipeline"
|
260
|
+
@running = true
|
261
|
+
trap("SIGINT") {self.stop}
|
262
|
+
started
|
263
|
+
while @running
|
264
|
+
run_in_once
|
265
|
+
run_out_once
|
266
|
+
sleep delay
|
267
|
+
end
|
268
|
+
stopped
|
269
|
+
log.info "#{name} stopped."
|
270
|
+
end
|
271
|
+
|
272
|
+
def stop
|
273
|
+
log.info "#{name} stopping..."
|
274
|
+
@running = false
|
275
|
+
Thread.main.run # i thought this would wake the thread from its sleep
|
276
|
+
# but it seems to have no effect.
|
277
|
+
end
|
240
278
|
|
241
279
|
# Override to process the event generated by the publisher before any other processing is done.
|
242
280
|
# Return false to veto the event.
|
@@ -287,7 +325,7 @@ module RubySync
|
|
287
325
|
|
288
326
|
def in_match event
|
289
327
|
log.debug "Default match rule - source path exists in vault"
|
290
|
-
vault.respond_to?(
|
328
|
+
vault.respond_to?('[]') and vault[event.source_path]
|
291
329
|
end
|
292
330
|
|
293
331
|
# If client_to_vault_map is defined (usually by map_client_to_vault)
|