rubysync 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/rubysync +312 -0
- data/examples/ar_client_webapp/README +182 -0
- data/examples/ar_client_webapp/Rakefile +10 -0
- data/examples/ar_client_webapp/app/controllers/application.rb +7 -0
- data/examples/ar_client_webapp/app/controllers/user_controller.rb +5 -0
- data/examples/ar_client_webapp/app/helpers/application_helper.rb +3 -0
- data/examples/ar_client_webapp/app/helpers/user_helper.rb +2 -0
- data/examples/ar_client_webapp/app/models/user.rb +2 -0
- data/examples/ar_client_webapp/config/boot.rb +45 -0
- data/examples/ar_client_webapp/config/database.yml +36 -0
- data/examples/ar_client_webapp/config/environment.rb +60 -0
- data/examples/ar_client_webapp/config/environments/development.rb +21 -0
- data/examples/ar_client_webapp/config/environments/production.rb +18 -0
- data/examples/ar_client_webapp/config/environments/test.rb +19 -0
- data/examples/ar_client_webapp/config/routes.rb +23 -0
- data/examples/ar_client_webapp/db/migrate/001_create_users.rb +13 -0
- data/examples/ar_client_webapp/db/schema.rb +13 -0
- data/examples/ar_client_webapp/doc/README_FOR_APP +2 -0
- data/examples/ar_client_webapp/public/404.html +30 -0
- data/examples/ar_client_webapp/public/500.html +30 -0
- data/examples/ar_client_webapp/public/dispatch.cgi +10 -0
- data/examples/ar_client_webapp/public/dispatch.fcgi +24 -0
- data/examples/ar_client_webapp/public/dispatch.rb +10 -0
- data/examples/ar_client_webapp/public/favicon.ico +0 -0
- data/examples/ar_client_webapp/public/images/rails.png +0 -0
- data/examples/ar_client_webapp/public/index.html +277 -0
- data/examples/ar_client_webapp/public/javascripts/application.js +2 -0
- data/examples/ar_client_webapp/public/javascripts/controls.js +833 -0
- data/examples/ar_client_webapp/public/javascripts/dragdrop.js +942 -0
- data/examples/ar_client_webapp/public/javascripts/effects.js +1088 -0
- data/examples/ar_client_webapp/public/javascripts/prototype.js +2515 -0
- data/examples/ar_client_webapp/public/robots.txt +1 -0
- data/examples/ar_client_webapp/script/about +3 -0
- data/examples/ar_client_webapp/script/breakpointer +3 -0
- data/examples/ar_client_webapp/script/console +3 -0
- data/examples/ar_client_webapp/script/destroy +3 -0
- data/examples/ar_client_webapp/script/generate +3 -0
- data/examples/ar_client_webapp/script/performance/benchmarker +3 -0
- data/examples/ar_client_webapp/script/performance/profiler +3 -0
- data/examples/ar_client_webapp/script/plugin +3 -0
- data/examples/ar_client_webapp/script/process/inspector +3 -0
- data/examples/ar_client_webapp/script/process/reaper +3 -0
- data/examples/ar_client_webapp/script/process/spawner +3 -0
- data/examples/ar_client_webapp/script/runner +3 -0
- data/examples/ar_client_webapp/script/server +3 -0
- data/examples/ar_client_webapp/test/fixtures/users.yml +5 -0
- data/examples/ar_client_webapp/test/functional/user_controller_test.rb +18 -0
- data/examples/ar_client_webapp/test/test_helper.rb +28 -0
- data/examples/ar_client_webapp/test/unit/user_test.rb +10 -0
- data/examples/ar_webapp/README +1 -0
- data/examples/ar_webapp/Rakefile +10 -0
- data/examples/ar_webapp/app/controllers/application.rb +7 -0
- data/examples/ar_webapp/app/controllers/hobbies_controller.rb +10 -0
- data/examples/ar_webapp/app/controllers/interests_controller.rb +9 -0
- data/examples/ar_webapp/app/controllers/people_controller.rb +14 -0
- data/examples/ar_webapp/app/controllers/ruby_sync_associations_controller.rb +10 -0
- data/examples/ar_webapp/app/helpers/application_helper.rb +3 -0
- data/examples/ar_webapp/app/models/hobby.rb +5 -0
- data/examples/ar_webapp/app/models/interest.rb +6 -0
- data/examples/ar_webapp/app/models/person.rb +9 -0
- data/examples/ar_webapp/app/models/ruby_sync_association.rb +5 -0
- data/examples/ar_webapp/app/models/ruby_sync_event.rb +9 -0
- data/examples/ar_webapp/app/models/ruby_sync_observer.rb +28 -0
- data/examples/ar_webapp/app/models/ruby_sync_operation.rb +20 -0
- data/examples/ar_webapp/app/models/ruby_sync_state.rb +2 -0
- data/examples/ar_webapp/app/models/ruby_sync_value.rb +7 -0
- data/examples/ar_webapp/app/views/layouts/application.rhtml +19 -0
- data/examples/ar_webapp/app/views/people/show.rhtml +18 -0
- data/examples/ar_webapp/config/boot.rb +45 -0
- data/examples/ar_webapp/config/database.yml +36 -0
- data/examples/ar_webapp/config/environment.rb +61 -0
- data/examples/ar_webapp/config/environments/development.rb +21 -0
- data/examples/ar_webapp/config/environments/production.rb +18 -0
- data/examples/ar_webapp/config/environments/test.rb +19 -0
- data/examples/ar_webapp/config/routes.rb +23 -0
- data/examples/ar_webapp/db/migrate/001_create_people.rb +12 -0
- data/examples/ar_webapp/db/migrate/002_create_interests.rb +12 -0
- data/examples/ar_webapp/db/migrate/003_create_hobbies.rb +11 -0
- data/examples/ar_webapp/db/migrate/004_create_ruby_sync_associations.rb +18 -0
- data/examples/ar_webapp/db/migrate/005_create_ruby_sync_events.rb +16 -0
- data/examples/ar_webapp/db/migrate/006_create_ruby_sync_operations.rb +15 -0
- data/examples/ar_webapp/db/migrate/007_create_ruby_sync_values.rb +12 -0
- data/examples/ar_webapp/db/migrate/008_ruby_sync_tracking.rb +16 -0
- data/examples/ar_webapp/db/migrate/009_create_ruby_sync_states.rb +10 -0
- data/examples/ar_webapp/db/schema.rb +56 -0
- data/examples/ar_webapp/doc/README_FOR_APP +2 -0
- data/examples/ar_webapp/public/404.html +30 -0
- data/examples/ar_webapp/public/500.html +30 -0
- data/examples/ar_webapp/public/dispatch.cgi +10 -0
- data/examples/ar_webapp/public/dispatch.fcgi +24 -0
- data/examples/ar_webapp/public/dispatch.rb +10 -0
- data/examples/ar_webapp/public/favicon.ico +0 -0
- data/examples/ar_webapp/public/images/rails.png +0 -0
- data/examples/ar_webapp/public/index.html +277 -0
- data/examples/ar_webapp/public/javascripts/application.js +2 -0
- data/examples/ar_webapp/public/javascripts/controls.js +833 -0
- data/examples/ar_webapp/public/javascripts/dragdrop.js +942 -0
- data/examples/ar_webapp/public/javascripts/effects.js +1088 -0
- data/examples/ar_webapp/public/javascripts/prototype.js +2515 -0
- data/examples/ar_webapp/public/robots.txt +1 -0
- data/examples/ar_webapp/script/about +3 -0
- data/examples/ar_webapp/script/breakpointer +3 -0
- data/examples/ar_webapp/script/console +3 -0
- data/examples/ar_webapp/script/destroy +3 -0
- data/examples/ar_webapp/script/generate +3 -0
- data/examples/ar_webapp/script/performance/benchmarker +3 -0
- data/examples/ar_webapp/script/performance/profiler +3 -0
- data/examples/ar_webapp/script/plugin +3 -0
- data/examples/ar_webapp/script/process/inspector +3 -0
- data/examples/ar_webapp/script/process/reaper +3 -0
- data/examples/ar_webapp/script/process/spawner +3 -0
- data/examples/ar_webapp/script/runner +3 -0
- data/examples/ar_webapp/script/server +3 -0
- data/examples/ar_webapp/test/fixtures/association_keys.yml +5 -0
- data/examples/ar_webapp/test/fixtures/hobbies.yml +5 -0
- data/examples/ar_webapp/test/fixtures/interests.yml +5 -0
- data/examples/ar_webapp/test/fixtures/people.yml +9 -0
- data/examples/ar_webapp/test/fixtures/ruby_sync_events.yml +5 -0
- data/examples/ar_webapp/test/fixtures/ruby_sync_operations.yml +5 -0
- data/examples/ar_webapp/test/fixtures/ruby_sync_states.yml +5 -0
- data/examples/ar_webapp/test/fixtures/ruby_sync_values.yml +5 -0
- data/examples/ar_webapp/test/test_helper.rb +28 -0
- data/examples/ar_webapp/test/unit/association_key_test.rb +8 -0
- data/examples/ar_webapp/test/unit/hobby_test.rb +10 -0
- data/examples/ar_webapp/test/unit/interest_test.rb +10 -0
- data/examples/ar_webapp/test/unit/person_test.rb +10 -0
- data/examples/ar_webapp/test/unit/ruby_sync_event_test.rb +12 -0
- data/examples/ar_webapp/test/unit/ruby_sync_observer_test.rb +57 -0
- data/examples/ar_webapp/test/unit/ruby_sync_operation_test.rb +10 -0
- data/examples/ar_webapp/test/unit/ruby_sync_state_test.rb +10 -0
- data/examples/ar_webapp/test/unit/ruby_sync_value_test.rb +10 -0
- data/examples/ims2/connectors/hr_db_connector.rb +8 -0
- data/examples/ims2/connectors/my_csv_connector.rb +12 -0
- data/examples/ims2/pipelines/hr_import_pipeline.rb +33 -0
- data/examples/my_ims/connectors/corp_directory_connector.rb +12 -0
- data/examples/my_ims/connectors/finance_connector.rb +7 -0
- data/examples/my_ims/connectors/hr_db_connector.rb +7 -0
- data/examples/my_ims/pipelines/finance_pipeline.rb +33 -0
- data/examples/my_ims/pipelines/hr_import_pipeline.rb +29 -0
- data/lib/ruby_sync/connectors/active_record_connector.rb +198 -0
- data/lib/ruby_sync/connectors/base_connector.rb +317 -0
- data/lib/ruby_sync/connectors/connector_event_processing.rb +78 -0
- data/lib/ruby_sync/connectors/csv_file_connector.rb +95 -0
- data/lib/ruby_sync/connectors/file_connector.rb +74 -0
- data/lib/ruby_sync/connectors/ldap_connector.rb +192 -0
- data/lib/ruby_sync/connectors/memory_connector.rb +185 -0
- data/lib/ruby_sync/event.rb +220 -0
- data/lib/ruby_sync/operation.rb +82 -0
- data/lib/ruby_sync/pipelines/base_pipeline.rb +360 -0
- data/lib/ruby_sync/util/metaid.rb +24 -0
- data/lib/ruby_sync/util/utilities.rb +115 -0
- data/lib/ruby_sync.rb +81 -0
- data/test/hashlike_tests.rb +111 -0
- data/test/ruby_sync_test.rb +48 -0
- data/test/test_active_record_vault.rb +113 -0
- data/test/test_csv_file_connector.rb +75 -0
- data/test/test_event.rb +40 -0
- data/test/test_ldap_connector.rb +89 -0
- data/test/test_memory_connectors.rb +40 -0
- data/test/ts_rubysync.rb +20 -0
- metadata +316 -0
@@ -0,0 +1,317 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (c) 2007 Ritchie Young. All rights reserved.
|
4
|
+
#
|
5
|
+
# This file is part of RubySync.
|
6
|
+
#
|
7
|
+
# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
|
8
|
+
# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
|
11
|
+
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
|
14
|
+
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
15
|
+
|
16
|
+
require 'ruby_sync/connectors/connector_event_processing'
|
17
|
+
|
18
|
+
module RubySync::Connectors
|
19
|
+
class BaseConnector
|
20
|
+
|
21
|
+
include RubySync::Utilities
|
22
|
+
include ConnectorEventProcessing
|
23
|
+
|
24
|
+
attr_accessor :once_only, :name, :is_vault
|
25
|
+
|
26
|
+
|
27
|
+
def initialize options={}
|
28
|
+
options = self.class.default_options.merge(options)
|
29
|
+
once_only = false
|
30
|
+
self.name = options[:name]
|
31
|
+
self.is_vault = options[:is_vault]
|
32
|
+
if is_vault && !can_act_as_vault?
|
33
|
+
raise Exception.new("#{self.class.name} can't act as an identity vault.")
|
34
|
+
end
|
35
|
+
options.each do |key, value|
|
36
|
+
if self.respond_to? "#{key}="
|
37
|
+
self.send("#{key}=", value)
|
38
|
+
else
|
39
|
+
log.debug "#{name}: doesn't respond to #{key}="
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
# Override this to return a string that will be included within the class definition of
|
46
|
+
# of configurations based on your connector.
|
47
|
+
def self.sample_config
|
48
|
+
end
|
49
|
+
|
50
|
+
# Override this to perform actions that must be performed the
|
51
|
+
# when the connector starts running. (Eg, opening network connections)
|
52
|
+
def started; end
|
53
|
+
|
54
|
+
# Subclasses must override this to
|
55
|
+
# interface with the external system and generate events.
|
56
|
+
# These events are yielded to the passed in block to process.
|
57
|
+
# This method will be called repeatedly until the connector is
|
58
|
+
# stopped.
|
59
|
+
def check; end
|
60
|
+
|
61
|
+
# Override this to perform actions that must be performed when
|
62
|
+
# the connector exits (eg closing network conections).
|
63
|
+
def stopped; end
|
64
|
+
|
65
|
+
|
66
|
+
# Call check repeatedly (or once if in once_only mode)
|
67
|
+
# to generate events.
|
68
|
+
# Should generally only be called by the pipeline to which it is attached.
|
69
|
+
def start
|
70
|
+
log.info "#{name}: Started"
|
71
|
+
@running = true
|
72
|
+
started()
|
73
|
+
while @running
|
74
|
+
check do |event|
|
75
|
+
if is_delete_echo?(event) || is_echo?(event)
|
76
|
+
log.debug "Ignoring echoed event"
|
77
|
+
else
|
78
|
+
call_if_exists :source_transform, event
|
79
|
+
yield(event)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
if once_only
|
84
|
+
log.debug "#{name}: Stopped"
|
85
|
+
@running = false
|
86
|
+
else
|
87
|
+
log.debug "#{name}: sleeping"
|
88
|
+
sleep 1
|
89
|
+
end
|
90
|
+
end
|
91
|
+
stopped
|
92
|
+
end
|
93
|
+
|
94
|
+
# Politely stop the connector.
|
95
|
+
def stop
|
96
|
+
log.info "#{name}: Attempting to stop"
|
97
|
+
@running = false
|
98
|
+
end
|
99
|
+
|
100
|
+
|
101
|
+
def is_vault?
|
102
|
+
@is_vault
|
103
|
+
end
|
104
|
+
|
105
|
+
# Returns the correct id for the given association
|
106
|
+
def path_for_association(association)
|
107
|
+
(is_vault?)?
|
108
|
+
path_for_foreign_key(association) : path_for_own_association_key(association.key)
|
109
|
+
end
|
110
|
+
|
111
|
+
|
112
|
+
# Returns the association key for the given path. Called if this connector is
|
113
|
+
# the client.
|
114
|
+
# The default implementation returns the path itself. If there is a more
|
115
|
+
# efficient key for looking up an entry in the client, override to return
|
116
|
+
# that instead.
|
117
|
+
def own_association_key_for(path)
|
118
|
+
path
|
119
|
+
end
|
120
|
+
|
121
|
+
|
122
|
+
# Returns the appropriate entry for the association key. This key will have been provided
|
123
|
+
# by a previous call to the association_key method.
|
124
|
+
# This will only be called on the client connector. It is not expected that the client will
|
125
|
+
# have to store this key.
|
126
|
+
def path_for_own_association_key(key)
|
127
|
+
key
|
128
|
+
end
|
129
|
+
|
130
|
+
# Returns the entry matching the association key. This is only called on the client.
|
131
|
+
def entry_for_own_association_key(key)
|
132
|
+
self[path_for_own_association_key(key)]
|
133
|
+
end
|
134
|
+
|
135
|
+
# True if there is an entry matching the association key. Only called on the client.
|
136
|
+
# Override if you have a quicker way of determining whether an entry exists for
|
137
|
+
# given key than retrieving the entry.
|
138
|
+
def has_entry_for_key?(key)
|
139
|
+
entry_for_own_association_key(key)
|
140
|
+
end
|
141
|
+
|
142
|
+
# Whether this connector is capable of acting as a vault.
|
143
|
+
# The vault is responsible for storing the association key of the client application
|
144
|
+
# and must be able to retrieve records for that association key.
|
145
|
+
# Typically, databases and directories can act as vaults, text documents and HR or finance
|
146
|
+
# applications probably can't.
|
147
|
+
# To enable a connector to act as a vault, define the following methods:
|
148
|
+
# => path_for_foreign_key(pipeline_id, key)
|
149
|
+
# => foreign_key_for(path)
|
150
|
+
# and associate_with_foreign_key(key, path).
|
151
|
+
def can_act_as_vault?
|
152
|
+
defined? associate and
|
153
|
+
defined? path_for_association and
|
154
|
+
defined? association_key_for and
|
155
|
+
defined? remove_association and
|
156
|
+
defined? associations_for
|
157
|
+
end
|
158
|
+
|
159
|
+
# TODO: These method signatures need to change to include a connector or pipeline id so that
|
160
|
+
# we can distinguish between foreign keys for the same record but different
|
161
|
+
# connectors/pipelines.
|
162
|
+
|
163
|
+
# def associate association, path
|
164
|
+
# end
|
165
|
+
#
|
166
|
+
# def path_for_association association
|
167
|
+
# end
|
168
|
+
#
|
169
|
+
# def association_key_for context, path
|
170
|
+
# end
|
171
|
+
#
|
172
|
+
# def associations_for path
|
173
|
+
# end
|
174
|
+
#
|
175
|
+
# def remove_association association
|
176
|
+
# end
|
177
|
+
|
178
|
+
# Return the association object given the association context and path.
|
179
|
+
# This should only be called on the vault.
|
180
|
+
def association_for(context, path)
|
181
|
+
raise "#{name} is not a vault." unless is_vault?
|
182
|
+
key = association_key_for context, path
|
183
|
+
RubySync::Association.new(context, key)
|
184
|
+
end
|
185
|
+
|
186
|
+
# Should only be called on the vault. Returns the entry associated with
|
187
|
+
# the association passed. Some connectors may wish to override this if
|
188
|
+
# they have a more efficient way of retrieving the record for a given
|
189
|
+
# association.
|
190
|
+
def find_associated association
|
191
|
+
path = path_for_association association
|
192
|
+
self[path]
|
193
|
+
end
|
194
|
+
|
195
|
+
# The context to be used to for all associations created where this
|
196
|
+
# connector is the client.
|
197
|
+
def association_context
|
198
|
+
self.name
|
199
|
+
end
|
200
|
+
|
201
|
+
# Attempts to delete non-existent items may occur due to echoing. Many systems won't be able to record
|
202
|
+
# the fact that an entry has been deleted by rubysync because after the delete, there is no entry left to
|
203
|
+
# record the information in. Therefore, they may issue a notification that the item has been deleted. This
|
204
|
+
# becomes an event and the connector won't know that it caused the delete. The story usually has a reasonably happy
|
205
|
+
# ending though.
|
206
|
+
# The inappropriate delete event is processed by the pipeline and a delete attempt is made on the
|
207
|
+
# datastore that actually triggered the original delete event in the first place. Most of the time, there will
|
208
|
+
# be no entry there for it to delete and it will fail harmlessly.
|
209
|
+
# Problems may arise, however, if the original delete event was the result of manipulation in the pipeline and
|
210
|
+
# the original entry is in fact supposed to stay there. For example, say a student in an enrolment system was marked
|
211
|
+
# as not enrolled anymore. This modify event is translated by the pipeline that connects to the identity vault to become
|
212
|
+
# a delete because only the enrolment system is interested in non-enrolled students. As the student is removed
|
213
|
+
# from the identity vault, a new delete event is generated targeted back and the enrolment system.
|
214
|
+
# If the pipeline has been configured to honour delete requests from the vault to the enrolment system then the
|
215
|
+
# students entry in the enrolment system would be deleted.
|
216
|
+
def is_delete_echo? event
|
217
|
+
false #TODO implement delete event caching
|
218
|
+
end
|
219
|
+
|
220
|
+
def is_echo? event; false end
|
221
|
+
|
222
|
+
# Called by unit tests to inject data
|
223
|
+
def test_add id, details
|
224
|
+
add id, details
|
225
|
+
end
|
226
|
+
|
227
|
+
# Called by unit tests to modify data
|
228
|
+
def test_modify id, details
|
229
|
+
modify id, details
|
230
|
+
end
|
231
|
+
|
232
|
+
# Called by unit tests to delete a record
|
233
|
+
def test_delete id
|
234
|
+
delete id
|
235
|
+
end
|
236
|
+
|
237
|
+
# Return an array of operations that would create the given record
|
238
|
+
# if applied to an empty hash.
|
239
|
+
def create_operations_for record
|
240
|
+
record.keys.map {|key| RubySync::Operation.new(:add, key, record[key])}
|
241
|
+
end
|
242
|
+
|
243
|
+
|
244
|
+
# Performs the given operations on the given record. The record is a
|
245
|
+
# Hash in which each key is a field name and each value is an array of
|
246
|
+
# values for that field.
|
247
|
+
# Operations is an Array of RubySync::Operation objects to be performed on the record.
|
248
|
+
def perform_operations operations, record={}
|
249
|
+
operations.each do |op|
|
250
|
+
unless op.instance_of? RubySync::Operation
|
251
|
+
log.warn "!!!!!!!!!! PROBLEM, DUMP FOLLOWS: !!!!!!!!!!!!!!"
|
252
|
+
p op
|
253
|
+
end
|
254
|
+
case op.type
|
255
|
+
when :add
|
256
|
+
if record[op.subject]
|
257
|
+
existing = record[op.subject].as_array
|
258
|
+
unless (existing & op.values).empty?
|
259
|
+
raise Exception.new("Attempt to add duplicate elements to #{name}")
|
260
|
+
end
|
261
|
+
record[op.subject] = existing + op.values
|
262
|
+
else
|
263
|
+
record[op.subject] = op.values
|
264
|
+
end
|
265
|
+
when :replace
|
266
|
+
record[op.subject] = op.values
|
267
|
+
when :delete
|
268
|
+
if value == nil || value == "" || value == []
|
269
|
+
record.delete(op.subject)
|
270
|
+
else
|
271
|
+
record[op.subject] -= values
|
272
|
+
end
|
273
|
+
else
|
274
|
+
raise Exception.new("Unknown operation '#{op.type}'")
|
275
|
+
end
|
276
|
+
end
|
277
|
+
return record
|
278
|
+
end
|
279
|
+
|
280
|
+
|
281
|
+
# Return an array of possible fields for this connector.
|
282
|
+
# Implementations should override this to query the datasource
|
283
|
+
# for possible fields.
|
284
|
+
def self.fields
|
285
|
+
nil
|
286
|
+
end
|
287
|
+
|
288
|
+
# Ensures that the named connector is loaded and returns its class object
|
289
|
+
def self.class_for connector_name
|
290
|
+
name = class_name_for connector_name
|
291
|
+
(name)? eval("::"+name) : nil
|
292
|
+
end
|
293
|
+
|
294
|
+
# Ensures that the named connector is loaded and returns its class name.
|
295
|
+
def self.class_name_for connector_name
|
296
|
+
filename = "#{connector_name}_connector"
|
297
|
+
class_name = filename.camelize
|
298
|
+
eval "defined? #{class_name}" or
|
299
|
+
$".include?(filename) or
|
300
|
+
require filename or
|
301
|
+
raise Exception.new("Can't find connector '#{filename}'")
|
302
|
+
class_name
|
303
|
+
end
|
304
|
+
|
305
|
+
private
|
306
|
+
|
307
|
+
def self.options options
|
308
|
+
@options = options
|
309
|
+
end
|
310
|
+
|
311
|
+
def self.default_options
|
312
|
+
@options ||= {}
|
313
|
+
end
|
314
|
+
|
315
|
+
end
|
316
|
+
end
|
317
|
+
|
@@ -0,0 +1,78 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Copyright (c) 2007 Ritchie Young. All rights reserved.
|
4
|
+
#
|
5
|
+
# This file is part of RubySync.
|
6
|
+
#
|
7
|
+
# RubySync is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License
|
8
|
+
# as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version.
|
9
|
+
#
|
10
|
+
# RubySync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied
|
11
|
+
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
12
|
+
#
|
13
|
+
# You should have received a copy of the GNU General Public License along with RubySync; if not, write to the
|
14
|
+
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA
|
15
|
+
|
16
|
+
|
17
|
+
|
18
|
+
module RubySync
|
19
|
+
module Connectors
|
20
|
+
|
21
|
+
# This is included into BaseConnector.
|
22
|
+
module ConnectorEventProcessing
|
23
|
+
|
24
|
+
def process(event)
|
25
|
+
case event.type
|
26
|
+
when :add: return perform_add(event)
|
27
|
+
when :delete: return perform_delete(event)
|
28
|
+
when :modify: return perform_modify(event)
|
29
|
+
else
|
30
|
+
raise Exception.new("#{name}: Unknown event type '#{event.type}' received")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Add a record to the connected store. If acting as vault, also associate with the attached association
|
35
|
+
# key for later retrieval. This implementation assumes that the target path is used. Connectors that
|
36
|
+
# make up their own key on creation of a record will need to override this.
|
37
|
+
def perform_add event
|
38
|
+
log.info "Adding '#{event.target_path}' to '#{name}'"
|
39
|
+
raise Exception.new("#{name}: Entry with path '#{event.target_path}' already exists, add failing.") if self[event.target_path]
|
40
|
+
if is_vault? && event.association && path_for_association(event.association)
|
41
|
+
raise Exception.new("#{name}: Association already in use. Add failing.")
|
42
|
+
end
|
43
|
+
call_if_exists(:target_transform, event)
|
44
|
+
if add(event.target_path, event.payload)
|
45
|
+
log.info "Add succeeded"
|
46
|
+
if is_vault?
|
47
|
+
if event.association
|
48
|
+
associate(event.association, event.target_path)
|
49
|
+
else
|
50
|
+
raise Exception.new("#{name}: No association key supplied to add.")
|
51
|
+
end
|
52
|
+
else
|
53
|
+
return own_association_key_for(event.target_path)
|
54
|
+
end
|
55
|
+
else
|
56
|
+
log.warn "Failed to add '#{event.target_path}' to '#{name}'"
|
57
|
+
return false
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def perform_delete event
|
62
|
+
raise Exception.new("#{name}: Delete of unassociated object. No action taken.") unless event.association
|
63
|
+
path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
|
64
|
+
log.info "Deleting '#{path}' from '#{name}'"
|
65
|
+
delete(path) or log.warn("#{name}: Attempted to delete non-existent entry '#{path}'\nMay be an echo of a delete from this connector, ignoring.")
|
66
|
+
return nil # don't want to create any new associations
|
67
|
+
end
|
68
|
+
|
69
|
+
def perform_modify event
|
70
|
+
path = (is_vault?)? path_for_association(event.association) : path_for_own_association_key(event.association.key)
|
71
|
+
raise Exception.new("#{name}: Attempted to modify non-existent entry '#{path}'") unless self[path]
|
72
|
+
call_if_exists(:target_transform, event)
|
73
|
+
modify path, event.payload
|
74
|
+
return (is_vault?)? nil : own_association_key_for(event.target_path)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Created by Ritchie Young on 2007-01-29.
|
4
|
+
# Copyright (c) 2007. All rights reserved.
|
5
|
+
|
6
|
+
require "csv"
|
7
|
+
require "ruby_sync/connectors/file_connector"
|
8
|
+
|
9
|
+
module RubySync
|
10
|
+
|
11
|
+
module Connectors
|
12
|
+
|
13
|
+
# Reads files containing Comma Separated Values from the in_path directory and
|
14
|
+
# treats each line as an incoming event.
|
15
|
+
#
|
16
|
+
# This Connector can't act as an identity vault.
|
17
|
+
class CsvFileConnector < RubySync::Connectors::FileConnector
|
18
|
+
|
19
|
+
attr_accessor :field_names # A list of names representing the namesspace for this connector
|
20
|
+
attr_accessor :path_field # The name of the field to use as the source_path
|
21
|
+
|
22
|
+
|
23
|
+
def initialize options={}
|
24
|
+
super options
|
25
|
+
@in_glob ||= '*.csv'
|
26
|
+
@out_extension ||= '.csv'
|
27
|
+
@field_names ||= []
|
28
|
+
@path_field ||= (@field_names.empty?)? 'field_0': @field_names[0]
|
29
|
+
end
|
30
|
+
|
31
|
+
# Called for each filename matching in_glob in in_path
|
32
|
+
# Yields a modify event for each row found in the file.
|
33
|
+
def check_file(filename)
|
34
|
+
CSV.open(filename, 'r') do |row|
|
35
|
+
if defined? field_name &&row.length > field_names.length
|
36
|
+
log.warn "#{name}: Row in file #{filename} exceeds defined field_names"
|
37
|
+
end
|
38
|
+
|
39
|
+
data = {}
|
40
|
+
row.each_index do |i|
|
41
|
+
field_name = (i < field_names.length)? field_names[i] : "field_#{i}"
|
42
|
+
data[field_name] = row[i].data
|
43
|
+
end
|
44
|
+
association_key = source_path = path_for(data)
|
45
|
+
yield RubySync::Event.modify(self, source_path, association_key, create_operations_for(data))
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.sample_config
|
50
|
+
return <<END
|
51
|
+
options(
|
52
|
+
:field_names=>['names', 'of', 'the', 'columns'],
|
53
|
+
:path_field=>'name_of_field_to_use_as_the_id',
|
54
|
+
:in_path=>'/directory/to/read/files/from',
|
55
|
+
:out_path=>'/directory/to/write/files/to',
|
56
|
+
:in_glob=>'*.csv',
|
57
|
+
:out_extension=>'.csv'
|
58
|
+
)
|
59
|
+
END
|
60
|
+
end
|
61
|
+
|
62
|
+
def self.fields
|
63
|
+
c = self.new
|
64
|
+
c.field_names and !c.field_names.empty? or
|
65
|
+
log.warn "Please set the field names in the connector config."
|
66
|
+
c.field_names || []
|
67
|
+
end
|
68
|
+
|
69
|
+
def write_record file, path, operations
|
70
|
+
record = perform_operations operations
|
71
|
+
line = CSV.generate_line(@field_names.map {|f| record[f]})
|
72
|
+
file.puts line
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
|
77
|
+
# Return the value to be used as the source_path for the event given the
|
78
|
+
# supplied row data.
|
79
|
+
def path_for(data)
|
80
|
+
if defined? @path_field
|
81
|
+
return data[@path_field]
|
82
|
+
end
|
83
|
+
return nil
|
84
|
+
end
|
85
|
+
|
86
|
+
# A file based system probably can't look data up so always return nil
|
87
|
+
# for lookup attempts
|
88
|
+
def [](path)
|
89
|
+
nil
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# Created by Ritchie Young on 2007-01-29.
|
4
|
+
# Copyright (c) 2007. All rights reserved.
|
5
|
+
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
module RubySync
|
9
|
+
|
10
|
+
module Connectors
|
11
|
+
|
12
|
+
# An abstract class that serves as the base for connectors
|
13
|
+
# that poll a filesystem directory for files and process them
|
14
|
+
# and/or write received events to a file.
|
15
|
+
class FileConnector < RubySync::Connectors::BaseConnector
|
16
|
+
|
17
|
+
attr_accessor :in_path # scan this directory for suitable files
|
18
|
+
attr_accessor :out_path # write received events to this directory
|
19
|
+
attr_accessor :out_extension # the file extension of files written to out_path
|
20
|
+
attr_accessor :in_glob # The filename glob for incoming files
|
21
|
+
|
22
|
+
|
23
|
+
def started
|
24
|
+
ensure_dir_exists @in_path
|
25
|
+
ensure_dir_exists @out_path
|
26
|
+
@out_extension ||= ".out"
|
27
|
+
end
|
28
|
+
|
29
|
+
def check(&blk)
|
30
|
+
unless in_glob
|
31
|
+
log.error "in_glob not set on file connector. No files will be processed"
|
32
|
+
return
|
33
|
+
end
|
34
|
+
log.info "#{name}: Scanning #{in_path} for #{in_glob} files..."
|
35
|
+
Dir.chdir(in_path) do |path|
|
36
|
+
Dir.glob(in_glob) do |filename|
|
37
|
+
log.info "#{name}: Processing '#{filename}'"
|
38
|
+
check_file filename, &blk
|
39
|
+
FileUtils.mv filename, "#{filename}.bak"
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# Called for each filename matching in_glob in in_path
|
45
|
+
def check_file(filename,&blk)
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
# TODO: Write to a temp file first and then move it to where it will
|
50
|
+
# be picked up by the receiving process.
|
51
|
+
# TODO: Make this use the same file for multiple records depending
|
52
|
+
# upon configuration of maximum lines per file and a timeout
|
53
|
+
def add path, operations
|
54
|
+
File.open(output_file_name, 'a') do |file|
|
55
|
+
write_record(file, path, operations)
|
56
|
+
end
|
57
|
+
return true
|
58
|
+
end
|
59
|
+
|
60
|
+
# Called to append a given record to an open file.
|
61
|
+
# Subclasses of FileConnector should override this.
|
62
|
+
def write_record file, path, operations
|
63
|
+
raise Exception.new("#{name} needs to implement 'write_record file, path, operations'")
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
# Generate a unique and appropriate filename within the given path
|
68
|
+
def output_file_name
|
69
|
+
File.join(@out_path, Time.now.strftime('%Y%m%d%H%M%S') + @out_extension)
|
70
|
+
end
|
71
|
+
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|