restforce-db 3.5.0 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/file_daemon.rb +10 -1
- data/lib/forked_process.rb +45 -0
- data/lib/restforce/db/client.rb +27 -0
- data/lib/restforce/db/command.rb +0 -4
- data/lib/restforce/db/field_processor.rb +48 -34
- data/lib/restforce/db/loggable.rb +43 -0
- data/lib/restforce/db/runner.rb +2 -0
- data/lib/restforce/db/task_manager.rb +96 -0
- data/lib/restforce/db/timestamp_cache.rb +20 -0
- data/lib/restforce/db/version.rb +1 -1
- data/lib/restforce/db/worker.rb +67 -76
- data/lib/restforce/db.rb +1 -36
- data/test/lib/restforce/db/timestamp_cache_test.rb +30 -0
- data/test/lib/restforce/db/worker_test.rb +5 -3
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c63f46aaa750d8936499c83d7e6f135e962c9d2e
|
4
|
+
data.tar.gz: 2838e8434cef8bb652fb3aac1251e4ce287f412d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0c5ac6edc50ab9ea35777bcfc677385feb7ea00734108b560012d249612e7c2f29dd65663d9390ceff16ec9272126f62014f282630f66d566128e0429cc0441c
|
7
|
+
data.tar.gz: 88c449e199a291dd8ea652d933dd51e0c07ed16e9cba034e297fd1cdeeb7f6aa69f7a9cde5cca5f93ea3125b1ee2b9fc17ab8f1e79d874f46adfc98ca020b139
|
data/lib/file_daemon.rb
CHANGED
@@ -15,12 +15,21 @@ module FileDaemon
|
|
15
15
|
# :nodoc:
|
16
16
|
module ClassMethods
|
17
17
|
|
18
|
+
# Public: Force-reopen all files at their current paths. Allows for rotation
|
19
|
+
# of log files outside of the context of an actual process fork.
|
20
|
+
#
|
21
|
+
# Returns nothing.
|
22
|
+
def reopen_files
|
23
|
+
before_fork
|
24
|
+
after_fork
|
25
|
+
end
|
26
|
+
|
18
27
|
# Public: Store the list of currently open file descriptors so that they
|
19
28
|
# may be reopened when a new process is spawned.
|
20
29
|
#
|
21
30
|
# Returns nothing.
|
22
31
|
def before_fork
|
23
|
-
@files_to_reopen
|
32
|
+
@files_to_reopen = ObjectSpace.each_object(File).reject(&:closed?)
|
24
33
|
end
|
25
34
|
|
26
35
|
# Public: Reopen all file descriptors that have been stored through the
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require "English"
|
2
|
+
|
3
|
+
# ForkedProcess exposes a small API for performing a block of code in a
|
4
|
+
# forked process, and relaying its output to another block.
|
5
|
+
class ForkedProcess
|
6
|
+
|
7
|
+
# Public: Define a callback which will be run in a forked process.
|
8
|
+
#
|
9
|
+
# Yields an IO object opened for writing when `run` is invoked.
|
10
|
+
# Returns nothing.
|
11
|
+
def write(&block)
|
12
|
+
@write_block = block
|
13
|
+
end
|
14
|
+
|
15
|
+
# Public: Define a callback which reads in the output from the forked
|
16
|
+
# process.
|
17
|
+
#
|
18
|
+
# Yields an IO object opened for reading when `run` is invoked.
|
19
|
+
# Returns nothing.
|
20
|
+
def read(&block)
|
21
|
+
@read_block = block
|
22
|
+
end
|
23
|
+
|
24
|
+
# Public: Fork a process, opening a pipe for IO and yielding the write and
|
25
|
+
# read components to the relevant blocks.
|
26
|
+
#
|
27
|
+
# Returns nothing.
|
28
|
+
def run
|
29
|
+
reader, writer = IO.pipe
|
30
|
+
|
31
|
+
pid = fork do
|
32
|
+
reader.close
|
33
|
+
@write_block.call(writer)
|
34
|
+
writer.close
|
35
|
+
exit!(0)
|
36
|
+
end
|
37
|
+
|
38
|
+
writer.close
|
39
|
+
@read_block.call(reader)
|
40
|
+
Process.wait(pid)
|
41
|
+
|
42
|
+
raise "Forked process did not exit successfully" unless $CHILD_STATUS.success?
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/lib/restforce/db/client.rb
CHANGED
@@ -7,6 +7,33 @@ module Restforce
|
|
7
7
|
# not yet supported by the base gem.
|
8
8
|
class Client < ::Restforce::Data::Client
|
9
9
|
|
10
|
+
# Public: Instantiate a new Restforce::DB::Client. Updates the middleware
|
11
|
+
# stack to account for some additional instrumentation and automatically
|
12
|
+
# retry timed out requests.
|
13
|
+
def initialize(**_)
|
14
|
+
super
|
15
|
+
|
16
|
+
# NOTE: By default, the Retry middleware will catch timeout exceptions,
|
17
|
+
# and retry up to two times. For more information, see:
|
18
|
+
# https://github.com/lostisland/faraday/blob/master/lib/faraday/request/retry.rb
|
19
|
+
middleware.insert(
|
20
|
+
-2,
|
21
|
+
Faraday::Request::Retry,
|
22
|
+
methods: [:get, :head, :options, :put, :patch, :delete],
|
23
|
+
)
|
24
|
+
|
25
|
+
middleware.insert_after(
|
26
|
+
Restforce::Middleware::InstanceURL,
|
27
|
+
FaradayMiddleware::Instrumentation,
|
28
|
+
name: "request.restforce_db",
|
29
|
+
)
|
30
|
+
|
31
|
+
middleware.insert_before(
|
32
|
+
FaradayMiddleware::Instrumentation,
|
33
|
+
Restforce::DB::Middleware::StoreRequestBody,
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
10
37
|
# Public: Get a list of Salesforce records which have been deleted between
|
11
38
|
# the specified times.
|
12
39
|
#
|
data/lib/restforce/db/command.rb
CHANGED
@@ -15,7 +15,6 @@ module Restforce
|
|
15
15
|
# args - A set of command line arguments to pass to the OptionParser.
|
16
16
|
def initialize(args)
|
17
17
|
@options = {
|
18
|
-
verbose: false,
|
19
18
|
pid_dir: Rails.root.join("tmp", "pids"),
|
20
19
|
config: Rails.root.join("config", "restforce-db.yml"),
|
21
20
|
tracker: Rails.root.join("config", ".restforce"),
|
@@ -104,9 +103,6 @@ module Restforce
|
|
104
103
|
opt.on("-t FILE", "--tracker FILE", "The file where run characteristics should be logged.") do |file|
|
105
104
|
@options[:tracker] = file
|
106
105
|
end
|
107
|
-
opt.on("-v", "--verbose", "Turn on noisy logging.") do
|
108
|
-
@options[:verbose] = true
|
109
|
-
end
|
110
106
|
end
|
111
107
|
end
|
112
108
|
|
@@ -10,19 +10,54 @@ module Restforce
|
|
10
10
|
# specific field.
|
11
11
|
RELATIONSHIP_MATCHER = /(.+)__r\./.freeze
|
12
12
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
13
|
+
class << self
|
14
|
+
|
15
|
+
# Public: Fetch the field metadata for all Salesforce models registered
|
16
|
+
# through mappings in the system. Useful to ensure that forked worker
|
17
|
+
# processes have access to all of the field metadata without the need
|
18
|
+
# for additional querying.
|
19
|
+
#
|
20
|
+
# Returns nothing.
|
21
|
+
def preload
|
22
|
+
Registry.each { |mapping| fetch(mapping.salesforce_model) }
|
23
|
+
end
|
24
|
+
|
25
|
+
# Public: Get a global cache with which to store/fetch the field
|
26
|
+
# metadata for each Salesforce Object Type.
|
27
|
+
#
|
28
|
+
# Returns a Hash.
|
29
|
+
def field_cache
|
30
|
+
@field_cache ||= {}
|
31
|
+
end
|
32
|
+
|
33
|
+
# Public: Get a collection of all fields for the passed Salesforce
|
34
|
+
# Object Type, with an indication of whether or not they are readable
|
35
|
+
# and writable for both create and update actions.
|
36
|
+
#
|
37
|
+
# sobject_type - A String name of an Object Type in Salesforce.
|
38
|
+
#
|
39
|
+
# Returns a Hash.
|
40
|
+
def fetch(sobject_type)
|
41
|
+
field_cache[sobject_type] ||= begin
|
42
|
+
fields = DB.client.describe(sobject_type).fields
|
43
|
+
|
44
|
+
fields.each_with_object({}) do |field, permissions|
|
45
|
+
permissions[field["name"]] = {
|
46
|
+
read: true,
|
47
|
+
create: field["createable"],
|
48
|
+
update: field["updateable"],
|
49
|
+
}
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
# Public: Clear out the global field cache.
|
55
|
+
#
|
56
|
+
# Returns nothing.
|
57
|
+
def reset
|
58
|
+
@field_cache = {}
|
59
|
+
end
|
20
60
|
|
21
|
-
# Internal: Clear out the global field cache.
|
22
|
-
#
|
23
|
-
# Returns nothing.
|
24
|
-
def self.reset
|
25
|
-
@field_cache = {}
|
26
61
|
end
|
27
62
|
|
28
63
|
# Public: Get a list of valid fields for a specific action from the passed
|
@@ -69,7 +104,7 @@ module Restforce
|
|
69
104
|
#
|
70
105
|
# Returns a Boolean.
|
71
106
|
def available?(sobject_type, field, action)
|
72
|
-
permissions =
|
107
|
+
permissions = self.class.fetch(sobject_type)[field]
|
73
108
|
return false unless permissions
|
74
109
|
|
75
110
|
permissions[action]
|
@@ -89,27 +124,6 @@ module Restforce
|
|
89
124
|
field =~ RELATIONSHIP_MATCHER
|
90
125
|
end
|
91
126
|
|
92
|
-
# Internal: Get a collection of all fields for the passed Salesforce
|
93
|
-
# SObject Type, with an indication of whether or not they are readable and
|
94
|
-
# writable for both create and update actions.
|
95
|
-
#
|
96
|
-
# sobject_type - A String name of an SObject Type in Salesforce.
|
97
|
-
#
|
98
|
-
# Returns a Hash.
|
99
|
-
def field_metadata(sobject_type)
|
100
|
-
self.class.field_cache[sobject_type] ||= begin
|
101
|
-
fields = Restforce::DB.client.describe(sobject_type).fields
|
102
|
-
|
103
|
-
fields.each_with_object({}) do |field, permissions|
|
104
|
-
permissions[field["name"]] = {
|
105
|
-
read: true,
|
106
|
-
create: field["createable"],
|
107
|
-
update: field["updateable"],
|
108
|
-
}
|
109
|
-
end
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
127
|
end
|
114
128
|
|
115
129
|
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Restforce
|
2
|
+
|
3
|
+
module DB
|
4
|
+
|
5
|
+
# Restforce::DB::Loggable defines shared behaviors for objects which
|
6
|
+
# need access to generic logging functionality.
|
7
|
+
module Loggable
|
8
|
+
|
9
|
+
# Public: Add a `logger` attribute to the object including this module.
|
10
|
+
#
|
11
|
+
# base - The object which is including the `Loggable` module.
|
12
|
+
def self.included(base)
|
13
|
+
base.send :attr_accessor, :logger
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
# Internal: Log the passed text at the specified level.
|
19
|
+
#
|
20
|
+
# text - The piece of text which should be logged for this worker.
|
21
|
+
# level - The level at which the text should be logged. Defaults to :info.
|
22
|
+
#
|
23
|
+
# Returns nothing.
|
24
|
+
def log(text, level = :info)
|
25
|
+
return unless logger
|
26
|
+
logger.send(level, text)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Internal: Log an error for the worker, outputting the entire error
|
30
|
+
# stacktrace and applying the appropriate log level.
|
31
|
+
#
|
32
|
+
# exception - An Exception object.
|
33
|
+
#
|
34
|
+
# Returns nothing.
|
35
|
+
def error(exception)
|
36
|
+
log exception, :error
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
data/lib/restforce/db/runner.rb
CHANGED
@@ -0,0 +1,96 @@
|
|
1
|
+
require "restforce/db/loggable"
|
2
|
+
require "restforce/db/task"
|
3
|
+
require "restforce/db/accumulator"
|
4
|
+
require "restforce/db/attacher"
|
5
|
+
require "restforce/db/associator"
|
6
|
+
require "restforce/db/cleaner"
|
7
|
+
require "restforce/db/collector"
|
8
|
+
require "restforce/db/initializer"
|
9
|
+
require "restforce/db/synchronizer"
|
10
|
+
|
11
|
+
module Restforce
|
12
|
+
|
13
|
+
# :nodoc:
|
14
|
+
module DB
|
15
|
+
|
16
|
+
# TaskMapping is a small data structure used to pass top-level task
|
17
|
+
# information through to a SynchronizationError when necessary.
|
18
|
+
TaskMapping = Struct.new(:id, :mapping)
|
19
|
+
|
20
|
+
# Restforce::DB::TaskManager defines the run sequence and invocation of each
|
21
|
+
# of the Restforce::DB::Task subclasses during a single processing loop for
|
22
|
+
# the top-level Worker object.
|
23
|
+
class TaskManager
|
24
|
+
|
25
|
+
include Loggable
|
26
|
+
|
27
|
+
# Public: Initialize a new Restforce::DB::TaskManager for a given runner
|
28
|
+
# state.
|
29
|
+
#
|
30
|
+
# runner - A Restforce::DB::Runner for a specific period of time.
|
31
|
+
# logger - A Logger object (optional).
|
32
|
+
def initialize(runner, logger: nil)
|
33
|
+
@runner = runner
|
34
|
+
@logger = logger
|
35
|
+
@changes = Hash.new { |h, k| h[k] = Accumulator.new }
|
36
|
+
end
|
37
|
+
|
38
|
+
# Public: Run each of the sync tasks in a defined order for the supplied
|
39
|
+
# runner's current state.
|
40
|
+
#
|
41
|
+
# Returns nothing.
|
42
|
+
def perform
|
43
|
+
Registry.each do |mapping|
|
44
|
+
run("CLEANING RECORDS", Cleaner, mapping)
|
45
|
+
run("ATTACHING RECORDS", Attacher, mapping)
|
46
|
+
run("PROPAGATING RECORDS", Initializer, mapping)
|
47
|
+
run("COLLECTING CHANGES", Collector, mapping)
|
48
|
+
end
|
49
|
+
|
50
|
+
# NOTE: We can only perform the synchronization after all record changes
|
51
|
+
# have been aggregated, so this second loop is necessary.
|
52
|
+
Registry.each do |mapping|
|
53
|
+
run("UPDATING ASSOCIATIONS", Associator, mapping)
|
54
|
+
run("APPLYING CHANGES", Synchronizer, mapping)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
# Internal: Log a description and response time for a specific named task.
|
61
|
+
#
|
62
|
+
# name - A String task name.
|
63
|
+
# task_class - A Restforce::DB::Task subclass.
|
64
|
+
# mapping - A Restforce::DB::Mapping.
|
65
|
+
#
|
66
|
+
# Returns a Boolean.
|
67
|
+
def run(name, task_class, mapping)
|
68
|
+
log " #{name} between #{mapping.database_model.name} and #{mapping.salesforce_model}"
|
69
|
+
runtime = Benchmark.realtime { task task_class, mapping }
|
70
|
+
log format(" FINISHED #{name} after %.4f", runtime)
|
71
|
+
|
72
|
+
true
|
73
|
+
rescue => e
|
74
|
+
error(e)
|
75
|
+
|
76
|
+
false
|
77
|
+
end
|
78
|
+
|
79
|
+
# Internal: Run the passed mapping through the supplied Task class.
|
80
|
+
#
|
81
|
+
# task_class - A Restforce::DB::Task subclass.
|
82
|
+
# mapping - A Restforce::DB::Mapping.
|
83
|
+
#
|
84
|
+
# Returns nothing.
|
85
|
+
def task(task_class, mapping)
|
86
|
+
task_class.new(mapping, @runner).run(@changes)
|
87
|
+
rescue Faraday::Error::ClientError => e
|
88
|
+
task_mapping = TaskMapping.new(task_class, mapping)
|
89
|
+
error SynchronizationError.new(e, task_mapping)
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -71,6 +71,26 @@ module Restforce
|
|
71
71
|
@cache = {}
|
72
72
|
end
|
73
73
|
|
74
|
+
# Public: Load the previous collection of cached timestamps from the
|
75
|
+
# passed readable object.
|
76
|
+
#
|
77
|
+
# io - An IO object opened for reading.
|
78
|
+
#
|
79
|
+
# Returns nothing.
|
80
|
+
def load_timestamps(io)
|
81
|
+
@cache = YAML.load(io.read) || {}
|
82
|
+
end
|
83
|
+
|
84
|
+
# Public: Dump the currently cached timestamps into the specified
|
85
|
+
# writable object.
|
86
|
+
#
|
87
|
+
# io - An IO object opened for writing.
|
88
|
+
#
|
89
|
+
# Returns nothing.
|
90
|
+
def dump_timestamps(io)
|
91
|
+
io.write(YAML.dump(@cache))
|
92
|
+
end
|
93
|
+
|
74
94
|
private
|
75
95
|
|
76
96
|
# Internal: Get a unique cache key for the passed instance.
|
data/lib/restforce/db/version.rb
CHANGED
data/lib/restforce/db/worker.rb
CHANGED
@@ -1,23 +1,29 @@
|
|
1
1
|
require "file_daemon"
|
2
|
+
require "forked_process"
|
3
|
+
require "restforce/db/task_manager"
|
4
|
+
require "restforce/db/loggable"
|
2
5
|
|
3
6
|
module Restforce
|
4
7
|
|
5
|
-
# :nodoc:
|
6
8
|
module DB
|
7
9
|
|
8
|
-
# TaskMapping is a small data structure used to pass top-level task
|
9
|
-
# information through to a SynchronizationError when necessary.
|
10
|
-
TaskMapping = Struct.new(:id, :mapping)
|
11
|
-
|
12
10
|
# Restforce::DB::Worker represents the primary polling loop through which
|
13
11
|
# all record synchronization occurs.
|
14
12
|
class Worker
|
15
13
|
|
16
14
|
include FileDaemon
|
15
|
+
include Loggable
|
17
16
|
|
18
17
|
DEFAULT_INTERVAL = 5
|
19
18
|
DEFAULT_DELAY = 1
|
20
19
|
|
20
|
+
# TERM and INT signals should trigger a graceful shutdown.
|
21
|
+
GRACEFUL_SHUTDOWN_SIGNALS = %w(TERM INT).freeze
|
22
|
+
|
23
|
+
# HUP and USR1 will reopen all files at their original paths, to
|
24
|
+
# accommodate log rotation.
|
25
|
+
ROTATION_SIGNALS = %w(HUP USR1).freeze
|
26
|
+
|
21
27
|
attr_accessor :logger, :tracker
|
22
28
|
|
23
29
|
# Public: Initialize a new Restforce::DB::Worker.
|
@@ -27,7 +33,6 @@ module Restforce
|
|
27
33
|
# interval - The maximum polling loop rest time.
|
28
34
|
# delay - The amount of time by which to offset queries.
|
29
35
|
# config - The path to a client configuration file.
|
30
|
-
# verbose - Display command line output? Defaults to false.
|
31
36
|
def initialize(options = {})
|
32
37
|
@options = options
|
33
38
|
@interval = @options.fetch(:interval) { DEFAULT_INTERVAL }
|
@@ -45,7 +50,10 @@ module Restforce
|
|
45
50
|
config.logger = logger
|
46
51
|
end
|
47
52
|
|
48
|
-
|
53
|
+
GRACEFUL_SHUTDOWN_SIGNALS.each { |signal| trap(signal) { stop } }
|
54
|
+
ROTATION_SIGNALS.each { |signal| trap(signal) { Worker.reopen_files } }
|
55
|
+
|
56
|
+
preload
|
49
57
|
|
50
58
|
loop do
|
51
59
|
runtime = Benchmark.realtime { perform }
|
@@ -66,28 +74,58 @@ module Restforce
|
|
66
74
|
|
67
75
|
private
|
68
76
|
|
77
|
+
# Internal: Populate the field cache for each Salesforce object in the
|
78
|
+
# defined mappings.
|
79
|
+
#
|
80
|
+
# NOTE: To work around thread-safety issues with Typheous (and possibly
|
81
|
+
# some other HTTP adapters, we need to fork our preloading to prevent
|
82
|
+
# intialization of our Client object in the context of the master Worker
|
83
|
+
# process.
|
84
|
+
#
|
85
|
+
# Returns a Hash.
|
86
|
+
def preload
|
87
|
+
forked = ForkedProcess.new
|
88
|
+
|
89
|
+
forked.write do |writer|
|
90
|
+
log "INITIALIZING..."
|
91
|
+
FieldProcessor.preload
|
92
|
+
YAML.dump(FieldProcessor.field_cache, writer)
|
93
|
+
end
|
94
|
+
|
95
|
+
forked.read do |reader|
|
96
|
+
FieldProcessor.field_cache.merge!(YAML.load(reader.read))
|
97
|
+
end
|
98
|
+
|
99
|
+
forked.run
|
100
|
+
end
|
101
|
+
|
69
102
|
# Internal: Perform the synchronization loop, recording the time that the
|
70
103
|
# run is performed so that future runs can pick up where the last run
|
71
104
|
# left off.
|
72
105
|
#
|
106
|
+
# NOTE: In order to keep our long-term memory usage in check, we fork a
|
107
|
+
# task manager to process the tasks for each synchronization loop. Once
|
108
|
+
# the subprocess dies, its memory can be reclaimed by the OS.
|
109
|
+
#
|
73
110
|
# Returns nothing.
|
74
111
|
def perform
|
112
|
+
reset!
|
113
|
+
|
75
114
|
track do
|
76
|
-
|
115
|
+
forked = ForkedProcess.new
|
116
|
+
|
117
|
+
forked.write do |writer|
|
118
|
+
Worker.after_fork
|
119
|
+
task_manager.perform
|
77
120
|
|
78
|
-
|
79
|
-
run("CLEANING RECORDS", Cleaner, mapping)
|
80
|
-
run("ATTACHING RECORDS", Attacher, mapping)
|
81
|
-
run("PROPAGATING RECORDS", Initializer, mapping)
|
82
|
-
run("COLLECTING CHANGES", Collector, mapping)
|
121
|
+
runner.dump_timestamps(writer)
|
83
122
|
end
|
84
123
|
|
85
|
-
|
86
|
-
|
87
|
-
Restforce::DB::Registry.each do |mapping|
|
88
|
-
run("UPDATING ASSOCIATIONS", Associator, mapping)
|
89
|
-
run("APPLYING CHANGES", Synchronizer, mapping)
|
124
|
+
forked.read do |reader|
|
125
|
+
runner.load_timestamps(reader)
|
90
126
|
end
|
127
|
+
|
128
|
+
forked.run
|
91
129
|
end
|
92
130
|
end
|
93
131
|
|
@@ -97,7 +135,15 @@ module Restforce
|
|
97
135
|
# Returns nothing.
|
98
136
|
def reset!
|
99
137
|
runner.tick!
|
100
|
-
|
138
|
+
Worker.before_fork
|
139
|
+
end
|
140
|
+
|
141
|
+
# Internal: Get a new TaskManager instance, which reflects the current
|
142
|
+
# runner state.
|
143
|
+
#
|
144
|
+
# Returns a Restforce::DB::TaskManager.
|
145
|
+
def task_manager
|
146
|
+
TaskManager.new(runner, logger: logger)
|
101
147
|
end
|
102
148
|
|
103
149
|
# Internal: Run the passed block, updating the tracker with the time at
|
@@ -115,9 +161,9 @@ module Restforce
|
|
115
161
|
log "SYNCHRONIZING"
|
116
162
|
end
|
117
163
|
|
118
|
-
yield
|
164
|
+
duration = Benchmark.realtime { yield }
|
165
|
+
log format("DONE after %.4f", duration)
|
119
166
|
|
120
|
-
log "DONE"
|
121
167
|
tracker.track(runtime)
|
122
168
|
else
|
123
169
|
yield
|
@@ -132,38 +178,6 @@ module Restforce
|
|
132
178
|
@runner ||= Runner.new(@options.fetch(:delay) { DEFAULT_DELAY })
|
133
179
|
end
|
134
180
|
|
135
|
-
# Internal: Log a description and response time for a specific named task.
|
136
|
-
#
|
137
|
-
# name - A String task name.
|
138
|
-
# task_class - A Restforce::DB::Task subclass.
|
139
|
-
# mapping - A Restforce::DB::Mapping.
|
140
|
-
#
|
141
|
-
# Returns a Boolean.
|
142
|
-
def run(name, task_class, mapping)
|
143
|
-
log " #{name} between #{mapping.database_model.name} and #{mapping.salesforce_model}"
|
144
|
-
runtime = Benchmark.realtime { task task_class, mapping }
|
145
|
-
log format(" COMPLETE after %.4f", runtime)
|
146
|
-
|
147
|
-
true
|
148
|
-
rescue => e
|
149
|
-
error(e)
|
150
|
-
|
151
|
-
false
|
152
|
-
end
|
153
|
-
|
154
|
-
# Internal: Run the passed mapping through the supplied Task class.
|
155
|
-
#
|
156
|
-
# task_class - A Restforce::DB::Task subclass.
|
157
|
-
# mapping - A Restforce::DB::Mapping.
|
158
|
-
#
|
159
|
-
# Returns nothing.
|
160
|
-
def task(task_class, mapping)
|
161
|
-
task_class.new(mapping, runner).run(@changes)
|
162
|
-
rescue Faraday::Error::ClientError => e
|
163
|
-
task_mapping = TaskMapping.new(task_class, mapping)
|
164
|
-
error SynchronizationError.new(e, task_mapping)
|
165
|
-
end
|
166
|
-
|
167
181
|
# Internal: Has this worker been instructed to stop?
|
168
182
|
#
|
169
183
|
# Returns a boolean.
|
@@ -171,29 +185,6 @@ module Restforce
|
|
171
185
|
@exit == true
|
172
186
|
end
|
173
187
|
|
174
|
-
# Internal: Log the passed text at the specified level.
|
175
|
-
#
|
176
|
-
# text - The piece of text which should be logged for this worker.
|
177
|
-
# level - The level at which the text should be logged. Defaults to :info.
|
178
|
-
#
|
179
|
-
# Returns nothing.
|
180
|
-
def log(text, level = :info)
|
181
|
-
puts text if @options[:verbose]
|
182
|
-
|
183
|
-
return unless logger
|
184
|
-
logger.send(level, text)
|
185
|
-
end
|
186
|
-
|
187
|
-
# Internal: Log an error for the worker, outputting the entire error
|
188
|
-
# stacktrace and applying the appropriate log level.
|
189
|
-
#
|
190
|
-
# exception - An Exception object.
|
191
|
-
#
|
192
|
-
# Returns nothing.
|
193
|
-
def error(exception)
|
194
|
-
logger.error(exception)
|
195
|
-
end
|
196
|
-
|
197
188
|
end
|
198
189
|
|
199
190
|
end
|
data/lib/restforce/db.rb
CHANGED
@@ -39,18 +39,10 @@ require "restforce/db/record_cache"
|
|
39
39
|
require "restforce/db/timestamp_cache"
|
40
40
|
require "restforce/db/runner"
|
41
41
|
|
42
|
-
require "restforce/db/task"
|
43
|
-
require "restforce/db/accumulator"
|
44
|
-
require "restforce/db/attacher"
|
45
42
|
require "restforce/db/adapter"
|
46
|
-
require "restforce/db/associator"
|
47
43
|
require "restforce/db/attribute_map"
|
48
|
-
require "restforce/db/cleaner"
|
49
|
-
require "restforce/db/collector"
|
50
|
-
require "restforce/db/initializer"
|
51
44
|
require "restforce/db/mapping"
|
52
45
|
require "restforce/db/model"
|
53
|
-
require "restforce/db/synchronizer"
|
54
46
|
require "restforce/db/tracker"
|
55
47
|
require "restforce/db/worker"
|
56
48
|
|
@@ -89,7 +81,7 @@ module Restforce
|
|
89
81
|
# Returns a Restforce::Data::Client instance.
|
90
82
|
def self.client
|
91
83
|
@client ||= begin
|
92
|
-
|
84
|
+
DB::Client.new(
|
93
85
|
username: configuration.username,
|
94
86
|
password: configuration.password,
|
95
87
|
security_token: configuration.security_token,
|
@@ -100,36 +92,9 @@ module Restforce
|
|
100
92
|
timeout: configuration.timeout,
|
101
93
|
adapter: configuration.adapter,
|
102
94
|
)
|
103
|
-
setup_middleware(client)
|
104
|
-
client
|
105
95
|
end
|
106
96
|
end
|
107
97
|
|
108
|
-
# Internal: Sets up the Restforce client's middleware handlers.
|
109
|
-
#
|
110
|
-
# Returns nothing.
|
111
|
-
def self.setup_middleware(client)
|
112
|
-
# NOTE: By default, the Retry middleware will catch timeout exceptions,
|
113
|
-
# and retry up to two times. For more information, see:
|
114
|
-
# https://github.com/lostisland/faraday/blob/master/lib/faraday/request/retry.rb
|
115
|
-
client.middleware.insert(
|
116
|
-
-2,
|
117
|
-
Faraday::Request::Retry,
|
118
|
-
methods: [:get, :head, :options, :put, :patch, :delete],
|
119
|
-
)
|
120
|
-
|
121
|
-
client.middleware.insert_after(
|
122
|
-
Restforce::Middleware::InstanceURL,
|
123
|
-
FaradayMiddleware::Instrumentation,
|
124
|
-
name: "request.restforce_db",
|
125
|
-
)
|
126
|
-
|
127
|
-
client.middleware.insert_before(
|
128
|
-
FaradayMiddleware::Instrumentation,
|
129
|
-
Restforce::DB::Middleware::StoreRequestBody,
|
130
|
-
)
|
131
|
-
end
|
132
|
-
|
133
98
|
# Public: Get the ID of the Salesforce user which is being used to access
|
134
99
|
# the Salesforce API.
|
135
100
|
#
|
@@ -84,4 +84,34 @@ describe Restforce::DB::TimestampCache do
|
|
84
84
|
end
|
85
85
|
end
|
86
86
|
|
87
|
+
describe "I/O operations" do
|
88
|
+
let(:io) { IO.pipe }
|
89
|
+
let(:reader) { io.first }
|
90
|
+
let(:writer) { io.last }
|
91
|
+
|
92
|
+
describe "#dump_timestamps" do
|
93
|
+
before do
|
94
|
+
cache.cache_timestamp instance
|
95
|
+
cache.dump_timestamps(writer)
|
96
|
+
writer.close
|
97
|
+
end
|
98
|
+
|
99
|
+
it "writes a YAML dump of the cache to the passed I/O object" do
|
100
|
+
expect(YAML.load(reader.read)).to_equal [record_type, id] => timestamp
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
describe "#load_timestamps" do
|
105
|
+
before do
|
106
|
+
YAML.dump({ [record_type, id] => timestamp }, writer)
|
107
|
+
writer.close
|
108
|
+
cache.load_timestamps(reader)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "reloads its internal cache from the passed I/O object" do
|
112
|
+
expect(cache.timestamp(instance)).to_equal timestamp
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
87
117
|
end
|
@@ -20,7 +20,8 @@ describe Restforce::DB::Worker do
|
|
20
20
|
|
21
21
|
## 1b. The record is synced to Salesforce.
|
22
22
|
worker.send :reset!
|
23
|
-
worker.send
|
23
|
+
manager = worker.send(:task_manager)
|
24
|
+
manager.send :task, Restforce::DB::Initializer, mapping
|
24
25
|
|
25
26
|
expect(database_record.reload).to_be :salesforce_id?
|
26
27
|
Salesforce.records << [salesforce_model, database_record.salesforce_id]
|
@@ -44,8 +45,9 @@ describe Restforce::DB::Worker do
|
|
44
45
|
# We sleep here to ensure we pick up our manual changes.
|
45
46
|
sleep 1 if VCR.current_cassette.recording?
|
46
47
|
worker.send :reset!
|
47
|
-
worker.send
|
48
|
-
|
48
|
+
manager = worker.send(:task_manager)
|
49
|
+
manager.send :task, Restforce::DB::Collector, mapping
|
50
|
+
manager.send :task, Restforce::DB::Synchronizer, mapping
|
49
51
|
end
|
50
52
|
end
|
51
53
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: restforce-db
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Horner
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -201,6 +201,7 @@ files:
|
|
201
201
|
- bin/setup
|
202
202
|
- circle.yml
|
203
203
|
- lib/file_daemon.rb
|
204
|
+
- lib/forked_process.rb
|
204
205
|
- lib/generators/restforce/install_generator.rb
|
205
206
|
- lib/generators/restforce/migration_generator.rb
|
206
207
|
- lib/generators/templates/config.yml
|
@@ -231,6 +232,7 @@ files:
|
|
231
232
|
- lib/restforce/db/instances/active_record.rb
|
232
233
|
- lib/restforce/db/instances/base.rb
|
233
234
|
- lib/restforce/db/instances/salesforce.rb
|
235
|
+
- lib/restforce/db/loggable.rb
|
234
236
|
- lib/restforce/db/mapping.rb
|
235
237
|
- lib/restforce/db/middleware/store_request_body.rb
|
236
238
|
- lib/restforce/db/model.rb
|
@@ -248,6 +250,7 @@ files:
|
|
248
250
|
- lib/restforce/db/synchronization_error.rb
|
249
251
|
- lib/restforce/db/synchronizer.rb
|
250
252
|
- lib/restforce/db/task.rb
|
253
|
+
- lib/restforce/db/task_manager.rb
|
251
254
|
- lib/restforce/db/timestamp_cache.rb
|
252
255
|
- lib/restforce/db/tracker.rb
|
253
256
|
- lib/restforce/db/version.rb
|