restforce-db 0.1.6 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -8
- data/lib/generators/templates/migration.rb.tt +2 -0
- data/lib/restforce/db/command.rb +17 -4
- data/lib/restforce/db/instances/active_record.rb +16 -0
- data/lib/restforce/db/instances/base.rb +12 -2
- data/lib/restforce/db/instances/salesforce.rb +8 -0
- data/lib/restforce/db/record_types/active_record.rb +7 -7
- data/lib/restforce/db/record_types/base.rb +17 -5
- data/lib/restforce/db/record_types/salesforce.rb +2 -1
- data/lib/restforce/db/synchronizer.rb +1 -1
- data/lib/restforce/db/tracker.rb +44 -0
- data/lib/restforce/db/version.rb +1 -1
- data/lib/restforce/db/worker.rb +47 -16
- data/lib/restforce/db.rb +3 -1
- data/test/cassettes/Restforce_DB_Synchronizer/_run/given_a_Salesforce_record_with_an_associated_database_record/when_synchronization_is_stale/updates_the_database_record.yml +197 -0
- data/test/cassettes/Restforce_DB_Synchronizer/_run/given_a_Salesforce_record_with_an_associated_database_record/when_synchronization_is_up-to-date/does_not_update_the_database_record.yml +234 -0
- data/test/lib/restforce/db/instances/active_record_test.rb +4 -0
- data/test/lib/restforce/db/record_types/active_record_test.rb +12 -8
- data/test/lib/restforce/db/synchronizer_test.rb +45 -14
- data/test/lib/restforce/db/tracker_test.rb +50 -0
- data/test/support/active_record.rb +4 -3
- metadata +6 -3
- data/test/cassettes/Restforce_DB_Synchronizer/_run/given_a_Salesforce_record_with_an_existing_record_in_the_database/updates_the_database_record.yml +0 -158
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9839ca71a54acd8b9da16ea06ebe5835abaec61d
|
4
|
+
data.tar.gz: 0395b99de5b72aa53b08ae0b3a758ed6d8e2b8d6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a73f240312804395c2469866b895531795e5c0aa25ef485649f783b1c2b04e2e53372f51ce29a35d17ee7bca9519af9593bb504b062fd8350ab1537c404545f3
|
7
|
+
data.tar.gz: 6633584716645f92206a9495d5bdaf25ee23c9ecc6aea28bbb50bad4d078465b0f0bd785744c3268efa4a0f329e0871b5416ec43feae1f412bbc287ce770d1d2
|
data/README.md
CHANGED
@@ -14,10 +14,6 @@ And then execute:
|
|
14
14
|
|
15
15
|
$ bundle
|
16
16
|
|
17
|
-
Or install it yourself as:
|
18
|
-
|
19
|
-
$ gem install restforce-db
|
20
|
-
|
21
17
|
## Usage
|
22
18
|
|
23
19
|
First, you'll want to install the default bin and configuration files, which is handled by the included Rails generator:
|
@@ -49,17 +45,19 @@ end
|
|
49
45
|
|
50
46
|
This will automatically register the model with an entry in the `Restforce::DB::RecordType` collection.
|
51
47
|
|
52
|
-
|
48
|
+
### Run the daemon
|
49
|
+
|
50
|
+
To actually perform the system synchronization, you'll want to run the binstub installed through the generator (see above). This will daemonize a process which loops repeatedly to continuously synchronize your database and Salesforce according to your established mappings.
|
53
51
|
|
54
|
-
$ bin/restforce-db start
|
52
|
+
$ bundle exec bin/restforce-db start
|
55
53
|
|
56
54
|
By default, this will load the credentials at the same location the generator installed them. You can explicitly pass the location of your configuration file with the `-c` option:
|
57
55
|
|
58
|
-
$ bin/restforce-db -c /path/to/my/config.yml start
|
56
|
+
$ bundle exec bin/restforce-db -c /path/to/my/config.yml start
|
59
57
|
|
60
58
|
For additional information and a full set of options, you can run:
|
61
59
|
|
62
|
-
$ bin/restforce-db -h
|
60
|
+
$ bundle exec bin/restforce-db -h
|
63
61
|
|
64
62
|
## Development
|
65
63
|
|
@@ -67,6 +65,11 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
67
65
|
|
68
66
|
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
69
67
|
|
68
|
+
## Caveats
|
69
|
+
|
70
|
+
- **Update Prioritization.** When synchronization occurs, newly-updated records in Salesforce are prioritized over newly-updated database records. This means that any changes to records in the database may be overwritten if changes were made to the Salesforce at the same time.
|
71
|
+
- **API Usage.** This gem performs most of its functionality via the Salesforce API (by way of the [`restforce`](https://github.com/ejholmes/restforce) gem). If you're at risk of hitting your Salesforce API limits, this may not be the right approach for you.
|
72
|
+
|
70
73
|
## Contributing
|
71
74
|
|
72
75
|
1. Fork it ( https://github.com/tablexi/restforce-db/fork )
|
@@ -4,6 +4,8 @@ class Add<%= class_name %>SalesforceBinding < ActiveRecord::Migration
|
|
4
4
|
# :nodoc:
|
5
5
|
def change
|
6
6
|
add_column :<%= table_name %>, :salesforce_id, :string
|
7
|
+
add_column :<%= table_name %>, :synchronized_at, :datetime
|
8
|
+
|
7
9
|
add_index :<%= table_name %>, :salesforce_id, unique: true
|
8
10
|
end
|
9
11
|
|
data/lib/restforce/db/command.rb
CHANGED
@@ -18,6 +18,7 @@ module Restforce
|
|
18
18
|
verbose: false,
|
19
19
|
pid_dir: Rails.root.join("tmp", "pids"),
|
20
20
|
config: Rails.root.join("config", "restforce-db.yml"),
|
21
|
+
tracker: Rails.root.join("config", ".restforce"),
|
21
22
|
logfile: Rails.root.join("log", "restforce-db.log"),
|
22
23
|
}
|
23
24
|
|
@@ -32,7 +33,7 @@ module Restforce
|
|
32
33
|
Dir.mkdir(dir) unless File.exist?(dir)
|
33
34
|
|
34
35
|
daemon_args = {
|
35
|
-
dir:
|
36
|
+
dir: dir,
|
36
37
|
dir_mode: :normal,
|
37
38
|
monitor: @monitor,
|
38
39
|
ARGV: @args,
|
@@ -56,12 +57,14 @@ module Restforce
|
|
56
57
|
Dir.chdir(Rails.root)
|
57
58
|
|
58
59
|
Restforce::DB::Worker.after_fork
|
59
|
-
Restforce::DB::Worker.logger ||= logger
|
60
60
|
|
61
61
|
worker = Restforce::DB::Worker.new(options)
|
62
|
+
worker.logger = logger
|
63
|
+
worker.tracker = tracker
|
64
|
+
|
62
65
|
worker.start
|
63
66
|
rescue => e
|
64
|
-
|
67
|
+
logger.fatal e
|
65
68
|
STDERR.puts e.message
|
66
69
|
exit 1
|
67
70
|
end
|
@@ -83,12 +86,15 @@ module Restforce
|
|
83
86
|
opt.on("-i N", "--interval N", "Amount of time to wait between synchronizations.") do |n|
|
84
87
|
@options[:interval] = n.to_i
|
85
88
|
end
|
86
|
-
opt.on("-l FILE
|
89
|
+
opt.on("-l FILE", "--logfile FILE", "The file where logging output should be captured.") do |file|
|
87
90
|
@options[:logfile] = file
|
88
91
|
end
|
89
92
|
opt.on("--pid-dir DIR", "The directory in which to store the pidfile.") do |dir|
|
90
93
|
@options[:pid_dir] = dir
|
91
94
|
end
|
95
|
+
opt.on("-t FILE", "--tracker FILE", "The file where run characteristics should be logged.") do |file|
|
96
|
+
@options[:tracker] = file
|
97
|
+
end
|
92
98
|
opt.on("-v", "--verbose", "Turn on noisy logging.") do
|
93
99
|
@options[:verbose] = true
|
94
100
|
end
|
@@ -107,6 +113,13 @@ module Restforce
|
|
107
113
|
end
|
108
114
|
end
|
109
115
|
|
116
|
+
# Internal: Get a Tracker instance to manage run characteristics.
|
117
|
+
#
|
118
|
+
# Returns a Restforce::DB::Tracker.
|
119
|
+
def tracker
|
120
|
+
@tracker ||= Tracker.new(@options[:tracker])
|
121
|
+
end
|
122
|
+
|
110
123
|
end
|
111
124
|
|
112
125
|
end
|
@@ -30,6 +30,22 @@ module Restforce
|
|
30
30
|
@record.updated_at
|
31
31
|
end
|
32
32
|
|
33
|
+
# Public: Get the time of the last synchronization update to this
|
34
|
+
# record.
|
35
|
+
#
|
36
|
+
# Returns a Time-compatible object.
|
37
|
+
def last_synchronize
|
38
|
+
@record.synchronized_at
|
39
|
+
end
|
40
|
+
|
41
|
+
# Public: Bump the synchronization timestamp on the record.
|
42
|
+
#
|
43
|
+
# Returns nothing.
|
44
|
+
def after_sync
|
45
|
+
@record.touch(:synchronized_at)
|
46
|
+
super
|
47
|
+
end
|
48
|
+
|
33
49
|
private
|
34
50
|
|
35
51
|
# Internal: Get a description of the expected attribute Hash format.
|
@@ -23,10 +23,11 @@ module Restforce
|
|
23
23
|
#
|
24
24
|
# attributes - A Hash mapping attribute names to values.
|
25
25
|
#
|
26
|
-
# Returns
|
26
|
+
# Returns self.
|
27
27
|
# Raises if the update fails for any reason.
|
28
28
|
def update!(attributes)
|
29
29
|
record.update!(attributes)
|
30
|
+
after_sync
|
30
31
|
end
|
31
32
|
|
32
33
|
# Public: Update the instance with attributes copied from the passed
|
@@ -36,7 +37,7 @@ module Restforce
|
|
36
37
|
# attributes corresponding to the configured mappings for this
|
37
38
|
# instance.
|
38
39
|
#
|
39
|
-
# Returns
|
40
|
+
# Returns self.
|
40
41
|
# Raises if the update fails for any reason.
|
41
42
|
def copy!(from_record)
|
42
43
|
update! @mapping.convert(conversion, from_record.attributes)
|
@@ -57,6 +58,15 @@ module Restforce
|
|
57
58
|
true
|
58
59
|
end
|
59
60
|
|
61
|
+
# Public: A hook which is performed after records are synchronized.
|
62
|
+
# Override this method in subclasses to inject generic behaviors into
|
63
|
+
# the record synchronization flow.
|
64
|
+
#
|
65
|
+
# Returns self.
|
66
|
+
def after_sync
|
67
|
+
self
|
68
|
+
end
|
69
|
+
|
60
70
|
end
|
61
71
|
|
62
72
|
end
|
@@ -28,6 +28,14 @@ module Restforce
|
|
28
28
|
Time.parse(@record.SystemModstamp)
|
29
29
|
end
|
30
30
|
|
31
|
+
# Public: Get the time of the last synchronization update to this
|
32
|
+
# record.
|
33
|
+
#
|
34
|
+
# Returns a Time-compatible object.
|
35
|
+
def last_synchronize
|
36
|
+
last_update
|
37
|
+
end
|
38
|
+
|
31
39
|
private
|
32
40
|
|
33
41
|
# Internal: Get a description of the expected attribute Hash format.
|
@@ -18,11 +18,10 @@ module Restforce
|
|
18
18
|
# Raises on any validation or database error.
|
19
19
|
def create!(from_record)
|
20
20
|
attributes = @mapping.convert(:database, from_record.attributes)
|
21
|
-
record = @record_type.create!(
|
22
|
-
attributes.merge(salesforce_id: from_record.id),
|
23
|
-
)
|
24
21
|
|
25
|
-
|
22
|
+
record = @record_type.create!(attributes.merge(salesforce_id: from_record.id))
|
23
|
+
|
24
|
+
Instances::ActiveRecord.new(record, @mapping).after_sync
|
26
25
|
end
|
27
26
|
|
28
27
|
# Public: Find the instance of this ActiveRecord model corresponding to
|
@@ -38,7 +37,8 @@ module Restforce
|
|
38
37
|
Instances::ActiveRecord.new(record, @mapping)
|
39
38
|
end
|
40
39
|
|
41
|
-
# Public: Iterate through all ActiveRecord records of
|
40
|
+
# Public: Iterate through all recently-updated ActiveRecord records of
|
41
|
+
# this type.
|
42
42
|
#
|
43
43
|
# options - A Hash of options which should be applied to the set of
|
44
44
|
# fetched records. Allowed options are:
|
@@ -50,7 +50,7 @@ module Restforce
|
|
50
50
|
# Yields a series of Restforce::DB::Instances::ActiveRecord instances.
|
51
51
|
# Returns nothing.
|
52
52
|
def each(options = {})
|
53
|
-
scope = @record_type
|
53
|
+
scope = @record_type.where("updated_at > synchronized_at OR synchronized_at IS NULL")
|
54
54
|
scope = scope.where("updated_at > ?", options[:after]) if options[:after]
|
55
55
|
scope = scope.where("updated_at < ?", options[:before]) if options[:before]
|
56
56
|
|
@@ -62,7 +62,7 @@ module Restforce
|
|
62
62
|
private
|
63
63
|
|
64
64
|
# Internal: Has this Salesforce record already been linked to a database
|
65
|
-
# record?
|
65
|
+
# record of this type?
|
66
66
|
#
|
67
67
|
# record - A Restforce::DB::Instances::Salesforce instance.
|
68
68
|
#
|
@@ -11,7 +11,7 @@ module Restforce
|
|
11
11
|
# Public: Initialize a new Restforce::DB::RecordTypes::Base.
|
12
12
|
#
|
13
13
|
# record_type - The name or class of the system record type.
|
14
|
-
# mapping -
|
14
|
+
# mapping - A Restforce::DB::Mapping.
|
15
15
|
def initialize(record_type, mapping = Mapping.new)
|
16
16
|
@record_type = record_type
|
17
17
|
@mapping = mapping
|
@@ -26,15 +26,27 @@ module Restforce
|
|
26
26
|
# Raises on any validation or external error.
|
27
27
|
def sync!(from_record)
|
28
28
|
if synced?(from_record)
|
29
|
-
|
30
|
-
record.copy!(from_record)
|
31
|
-
|
32
|
-
record
|
29
|
+
update!(from_record)
|
33
30
|
else
|
34
31
|
create!(from_record)
|
35
32
|
end
|
36
33
|
end
|
37
34
|
|
35
|
+
# Public: Update an existing record of this record type with the
|
36
|
+
# attributes from the passed record. Only applies changes if from_record
|
37
|
+
# has been more recently updated than the last record synchronization.
|
38
|
+
#
|
39
|
+
# from_record - A Restforce::DB::Instances::Base instance.
|
40
|
+
#
|
41
|
+
# Returns a Restforce::DB::Instances::Base instance.
|
42
|
+
# Raises on any validation or external error.
|
43
|
+
def update!(from_record)
|
44
|
+
record = find(from_record.id)
|
45
|
+
return record if from_record.last_update < record.last_synchronize
|
46
|
+
|
47
|
+
record.copy!(from_record)
|
48
|
+
end
|
49
|
+
|
38
50
|
end
|
39
51
|
|
40
52
|
end
|
@@ -19,7 +19,8 @@ module Restforce
|
|
19
19
|
def create!(from_record)
|
20
20
|
attributes = @mapping.convert(:salesforce, from_record.attributes)
|
21
21
|
record_id = DB.client.create!(@record_type, attributes)
|
22
|
-
|
22
|
+
|
23
|
+
from_record.update!(salesforce_id: record_id).after_sync
|
23
24
|
|
24
25
|
find(record_id)
|
25
26
|
end
|
@@ -19,7 +19,7 @@ module Restforce
|
|
19
19
|
# last_run_time - A Time object reflecting the time of the most
|
20
20
|
# recent synchronization run. Runs will only
|
21
21
|
# synchronize data more recent than this stamp.
|
22
|
-
def initialize(database_record_type, salesforce_record_type, last_run_time =
|
22
|
+
def initialize(database_record_type, salesforce_record_type, last_run_time = DB.last_run)
|
23
23
|
@database_record_type = database_record_type
|
24
24
|
@salesforce_record_type = salesforce_record_type
|
25
25
|
@last_run = last_run_time
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Restforce
|
2
|
+
|
3
|
+
module DB
|
4
|
+
|
5
|
+
# Restforce::DB::Tracker encapsulates a minimal API to track and configure
|
6
|
+
# synchronization runtimes. It allows Restforce::DB to persist a "last
|
7
|
+
# successful sync" timestamp.
|
8
|
+
class Tracker
|
9
|
+
|
10
|
+
attr_reader :last_run
|
11
|
+
|
12
|
+
# Public: Initialize a Restforce::DB::Tracker. Sets a last_run timestamp
|
13
|
+
# on Restforce::DB if the supplied tracking file already contains a stamp.
|
14
|
+
#
|
15
|
+
# file_path - The Path to the tracking file.
|
16
|
+
def initialize(file_path)
|
17
|
+
@file = File.open(file_path, "a+")
|
18
|
+
|
19
|
+
timestamp = @file.read
|
20
|
+
return if timestamp.empty?
|
21
|
+
|
22
|
+
@last_run = Time.parse(timestamp)
|
23
|
+
Restforce::DB.last_run = @last_run
|
24
|
+
end
|
25
|
+
|
26
|
+
# Public: Persist the passed time in the tracker file.
|
27
|
+
#
|
28
|
+
# time - A Time object.
|
29
|
+
#
|
30
|
+
# Returns nothing.
|
31
|
+
def track(time)
|
32
|
+
@last_run = time
|
33
|
+
|
34
|
+
@file.truncate(0)
|
35
|
+
@file.print(time.utc.iso8601)
|
36
|
+
@file.flush
|
37
|
+
@file.rewind
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
data/lib/restforce/db/version.rb
CHANGED
data/lib/restforce/db/worker.rb
CHANGED
@@ -10,8 +10,6 @@ module Restforce
|
|
10
10
|
|
11
11
|
class << self
|
12
12
|
|
13
|
-
attr_accessor :logger, :interval
|
14
|
-
|
15
13
|
# Public: Store the list of currently open file descriptors so that they
|
16
14
|
# may be reopened when a new process is spawned.
|
17
15
|
#
|
@@ -41,6 +39,8 @@ module Restforce
|
|
41
39
|
|
42
40
|
end
|
43
41
|
|
42
|
+
attr_accessor :logger, :tracker
|
43
|
+
|
44
44
|
# Public: Initialize a new Restforce::DB::Worker.
|
45
45
|
#
|
46
46
|
# options - A Hash of options to configure the worker's run. Currently
|
@@ -50,7 +50,7 @@ module Restforce
|
|
50
50
|
# verbose - Display command line output? Defaults to false.
|
51
51
|
def initialize(options = {})
|
52
52
|
@verbose = options.fetch(:verbose) { false }
|
53
|
-
|
53
|
+
@interval = options.fetch(:interval) { DEFAULT_INTERVAL }
|
54
54
|
|
55
55
|
Restforce::DB.configure { |config| config.parse(options[:config]) }
|
56
56
|
end
|
@@ -72,16 +72,10 @@ module Restforce
|
|
72
72
|
end
|
73
73
|
|
74
74
|
loop do
|
75
|
-
|
76
|
-
|
77
|
-
runtime = Benchmark.realtime do
|
78
|
-
Restforce::DB::RecordType.each do |name, record_type|
|
79
|
-
synchronize name, record_type
|
80
|
-
end
|
81
|
-
end
|
75
|
+
runtime = Benchmark.realtime { perform }
|
82
76
|
|
83
|
-
if runtime <
|
84
|
-
sleep(
|
77
|
+
if runtime < @interval && !stop?
|
78
|
+
sleep(@interval - runtime)
|
85
79
|
end
|
86
80
|
|
87
81
|
break if stop?
|
@@ -98,6 +92,43 @@ module Restforce
|
|
98
92
|
|
99
93
|
private
|
100
94
|
|
95
|
+
# Internal: Perform the synchronization loop, recording the time that the
|
96
|
+
# run is performed so that future runs can pick up where the last run
|
97
|
+
# left off.
|
98
|
+
#
|
99
|
+
# Returns nothing.
|
100
|
+
def perform
|
101
|
+
track do
|
102
|
+
Restforce::DB::RecordType.each do |name, record_type|
|
103
|
+
synchronize name, record_type
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# Internal: Run the passed block, updating the tracker with the time at
|
109
|
+
# which the run was initiated.
|
110
|
+
#
|
111
|
+
# Yields to a passed block.
|
112
|
+
# Returns nothing.
|
113
|
+
def track
|
114
|
+
if tracker
|
115
|
+
runtime = Time.now
|
116
|
+
|
117
|
+
if tracker.last_run
|
118
|
+
log "SYNCHRONIZING from #{tracker.last_run.iso8601}"
|
119
|
+
else
|
120
|
+
log "SYNCHRONIZING"
|
121
|
+
end
|
122
|
+
|
123
|
+
yield
|
124
|
+
|
125
|
+
log "DONE"
|
126
|
+
tracker.track(runtime)
|
127
|
+
else
|
128
|
+
yield
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
101
132
|
# Internal: Synchronize the objects in the database and Salesforce
|
102
133
|
# corresponding to the passed record type.
|
103
134
|
#
|
@@ -106,9 +137,9 @@ module Restforce
|
|
106
137
|
#
|
107
138
|
# Returns a Boolean.
|
108
139
|
def synchronize(name, record_type)
|
109
|
-
log "
|
140
|
+
log " SYNCHRONIZE #{name}"
|
110
141
|
runtime = Benchmark.realtime { record_type.synchronize }
|
111
|
-
log format("
|
142
|
+
log format(" COMPLETE after %.4f", runtime)
|
112
143
|
|
113
144
|
return true
|
114
145
|
rescue => e
|
@@ -133,8 +164,8 @@ module Restforce
|
|
133
164
|
def log(text, level = :info)
|
134
165
|
puts text if @verbose
|
135
166
|
|
136
|
-
return unless
|
137
|
-
|
167
|
+
return unless logger
|
168
|
+
logger.send(level, text)
|
138
169
|
end
|
139
170
|
|
140
171
|
# Internal: Log an error for the worker, outputting the entire error
|
data/lib/restforce/db.rb
CHANGED
@@ -16,8 +16,9 @@ require "restforce/db/record_types/salesforce"
|
|
16
16
|
|
17
17
|
require "restforce/db/mapping"
|
18
18
|
require "restforce/db/model"
|
19
|
-
require "restforce/db/synchronizer"
|
20
19
|
require "restforce/db/record_type"
|
20
|
+
require "restforce/db/synchronizer"
|
21
|
+
require "restforce/db/tracker"
|
21
22
|
require "restforce/db/worker"
|
22
23
|
|
23
24
|
module Restforce
|
@@ -28,6 +29,7 @@ module Restforce
|
|
28
29
|
|
29
30
|
class << self
|
30
31
|
|
32
|
+
attr_accessor :last_run
|
31
33
|
attr_writer :configuration
|
32
34
|
|
33
35
|
end
|