restforce-db 0.1.6 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2a376fa3d3ec57765c0d869be30dbd381e637559
-  data.tar.gz: 686f7970d90cb52377daeefab21f6e0b91c2d03c
+  metadata.gz: 9839ca71a54acd8b9da16ea06ebe5835abaec61d
+  data.tar.gz: 0395b99de5b72aa53b08ae0b3a758ed6d8e2b8d6
 SHA512:
-  metadata.gz: f377bfbcc127e815625f81908d2ab4ac509c6bce489552808dfcf11af3236ec8abdd299661c58fec0fe84ec0fcb3824678fe41ee51aaed274522fc03147b10ef
-  data.tar.gz: aeafbb041eef87730d18558bb897e5606cf54b4f8dbb379bcb4b08e7d3a789a703a3da99c9c22669588c6d78f59624344b0d88236c3629c02b0c5f1dadd6a0f8
+  metadata.gz: a73f240312804395c2469866b895531795e5c0aa25ef485649f783b1c2b04e2e53372f51ce29a35d17ee7bca9519af9593bb504b062fd8350ab1537c404545f3
+  data.tar.gz: 6633584716645f92206a9495d5bdaf25ee23c9ecc6aea28bbb50bad4d078465b0f0bd785744c3268efa4a0f329e0871b5416ec43feae1f412bbc287ce770d1d2

data/README.md

@@ -14,10 +14,6 @@ And then execute:
 
     $ bundle
 
-Or install it yourself as:
-
-    $ gem install restforce-db
-
 ## Usage
 
 First, you'll want to install the default bin and configuration files, which is handled by the included Rails generator:
@@ -49,17 +45,19 @@ end
 
 This will automatically register the model with an entry in the `Restforce::DB::RecordType` collection.
 
-To run the worker, you'll want to run the binstub installed through the generator (see above). Then you can run the self-daemonizing executable.
+### Run the daemon
+
+To actually perform the system synchronization, you'll want to run the binstub installed through the generator (see above). This will daemonize a process which loops repeatedly to continuously synchronize your database and Salesforce according to your established mappings.
 
-        $ bin/restforce-db start
+        $ bundle exec bin/restforce-db start
 
 By default, this will load the credentials at the same location the generator installed them. You can explicitly pass the location of your configuration file with the `-c` option:
 
-        $ bin/restforce-db -c /path/to/my/config.yml start
+        $ bundle exec bin/restforce-db -c /path/to/my/config.yml start
 
 For additional information and a full set of options, you can run:
 
-        $ bin/restforce-db -h
+        $ bundle exec bin/restforce-db -h
 
 ## Development
 
@@ -67,6 +65,11 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+## Caveats
+
+- **Update Prioritization.** When synchronization occurs, newly-updated records in Salesforce are prioritized over newly-updated database records. This means that any changes to records in the database may be overwritten if changes were made to the Salesforce at the same time.
+- **API Usage.** This gem performs most of its functionality via the Salesforce API (by way of the [`restforce`](https://github.com/ejholmes/restforce) gem). If you're at risk of hitting your Salesforce API limits, this may not be the right approach for you.
+
 ## Contributing
 
 1. Fork it ( https://github.com/tablexi/restforce-db/fork )

data/lib/generators/templates/migration.rb.tt

@@ -4,6 +4,8 @@ class Add<%= class_name %>SalesforceBinding < ActiveRecord::Migration
   # :nodoc:
   def change
     add_column :<%= table_name %>, :salesforce_id, :string
+    add_column :<%= table_name %>, :synchronized_at, :datetime
+
     add_index :<%= table_name %>, :salesforce_id, unique: true
   end
 

data/lib/restforce/db/command.rb

@@ -18,6 +18,7 @@ module Restforce
           verbose: false,
           pid_dir: Rails.root.join("tmp", "pids"),
           config:  Rails.root.join("config", "restforce-db.yml"),
+          tracker: Rails.root.join("config", ".restforce"),
           logfile: Rails.root.join("log", "restforce-db.log"),
         }
 
@@ -32,7 +33,7 @@ module Restforce
         Dir.mkdir(dir) unless File.exist?(dir)
 
         daemon_args = {
-          dir: @options[:pid_dir],
+          dir: dir,
           dir_mode: :normal,
           monitor: @monitor,
           ARGV: @args,
@@ -56,12 +57,14 @@ module Restforce
         Dir.chdir(Rails.root)
 
         Restforce::DB::Worker.after_fork
-        Restforce::DB::Worker.logger ||= logger
 
         worker = Restforce::DB::Worker.new(options)
+        worker.logger = logger
+        worker.tracker = tracker
+
         worker.start
       rescue => e
-        Restforce::DB::Worker.logger.fatal e
+        logger.fatal e
         STDERR.puts e.message
         exit 1
       end
@@ -83,12 +86,15 @@ module Restforce
           opt.on("-i N", "--interval N", "Amount of time to wait between synchronizations.") do |n|
             @options[:interval] = n.to_i
           end
-          opt.on("-l FILE ", "--logfile FILE", "The file where logging output should be captured.") do |file|
+          opt.on("-l FILE", "--logfile FILE", "The file where logging output should be captured.") do |file|
             @options[:logfile] = file
           end
           opt.on("--pid-dir DIR", "The directory in which to store the pidfile.") do |dir|
             @options[:pid_dir] = dir
           end
+          opt.on("-t FILE", "--tracker FILE", "The file where run characteristics should be logged.") do |file|
+            @options[:tracker] = file
+          end
           opt.on("-v", "--verbose", "Turn on noisy logging.") do
             @options[:verbose] = true
           end
@@ -107,6 +113,13 @@ module Restforce
         end
       end
 
+      # Internal: Get a Tracker instance to manage run characteristics.
+      #
+      # Returns a Restforce::DB::Tracker.
+      def tracker
+        @tracker ||= Tracker.new(@options[:tracker])
+      end
+
     end
 
   end

data/lib/restforce/db/instances/active_record.rb

@@ -30,6 +30,22 @@ module Restforce
           @record.updated_at
         end
 
+        # Public: Get the time of the last synchronization update to this
+        # record.
+        #
+        # Returns a Time-compatible object.
+        def last_synchronize
+          @record.synchronized_at
+        end
+
+        # Public: Bump the synchronization timestamp on the record.
+        #
+        # Returns nothing.
+        def after_sync
+          @record.touch(:synchronized_at)
+          super
+        end
+
         private
 
         # Internal: Get a description of the expected attribute Hash format.

data/lib/restforce/db/instances/base.rb

@@ -23,10 +23,11 @@ module Restforce
         #
         # attributes - A Hash mapping attribute names to values.
         #
-        # Returns a truthy value.
+        # Returns self.
         # Raises if the update fails for any reason.
         def update!(attributes)
           record.update!(attributes)
+          after_sync
         end
 
         # Public: Update the instance with attributes copied from the passed
@@ -36,7 +37,7 @@ module Restforce
         #          attributes corresponding to the configured mappings for this
         #          instance.
         #
-        # Returns a truthy value.
+        # Returns self.
         # Raises if the update fails for any reason.
         def copy!(from_record)
           update! @mapping.convert(conversion, from_record.attributes)
@@ -57,6 +58,15 @@ module Restforce
           true
         end
 
+        # Public: A hook which is performed after records are synchronized.
+        # Override this method in subclasses to inject generic behaviors into
+        # the record synchronization flow.
+        #
+        # Returns self.
+        def after_sync
+          self
+        end
+
       end
 
     end

data/lib/restforce/db/instances/salesforce.rb

@@ -28,6 +28,14 @@ module Restforce
           Time.parse(@record.SystemModstamp)
         end
 
+        # Public: Get the time of the last synchronization update to this
+        # record.
+        #
+        # Returns a Time-compatible object.
+        def last_synchronize
+          last_update
+        end
+
         private
 
         # Internal: Get a description of the expected attribute Hash format.

data/lib/restforce/db/record_types/active_record.rb

@@ -18,11 +18,10 @@ module Restforce
         # Raises on any validation or database error.
         def create!(from_record)
           attributes = @mapping.convert(:database, from_record.attributes)
-          record = @record_type.create!(
-            attributes.merge(salesforce_id: from_record.id),
-          )
 
-          Instances::ActiveRecord.new(record, @mapping)
+          record = @record_type.create!(attributes.merge(salesforce_id: from_record.id))
+
+          Instances::ActiveRecord.new(record, @mapping).after_sync
         end
 
         # Public: Find the instance of this ActiveRecord model corresponding to
@@ -38,7 +37,8 @@ module Restforce
           Instances::ActiveRecord.new(record, @mapping)
         end
 
-        # Public: Iterate through all ActiveRecord records of this type.
+        # Public: Iterate through all recently-updated ActiveRecord records of
+        # this type.
         #
         # options - A Hash of options which should be applied to the set of
         #           fetched records. Allowed options are:
@@ -50,7 +50,7 @@ module Restforce
         # Yields a series of Restforce::DB::Instances::ActiveRecord instances.
         # Returns nothing.
         def each(options = {})
-          scope = @record_type
+          scope = @record_type.where("updated_at > synchronized_at OR synchronized_at IS NULL")
           scope = scope.where("updated_at > ?", options[:after]) if options[:after]
           scope = scope.where("updated_at < ?", options[:before]) if options[:before]
 
@@ -62,7 +62,7 @@ module Restforce
         private
 
         # Internal: Has this Salesforce record already been linked to a database
-        # record?
+        # record of this type?
         #
         # record - A Restforce::DB::Instances::Salesforce instance.
         #

data/lib/restforce/db/record_types/base.rb

@@ -11,7 +11,7 @@ module Restforce
         # Public: Initialize a new Restforce::DB::RecordTypes::Base.
         #
         # record_type - The name or class of the system record type.
-        # mapping     - An instance of Restforce::DB::Mapping.
+        # mapping     - A Restforce::DB::Mapping.
         def initialize(record_type, mapping = Mapping.new)
           @record_type = record_type
           @mapping = mapping
@@ -26,15 +26,27 @@ module Restforce
         # Raises on any validation or external error.
         def sync!(from_record)
           if synced?(from_record)
-            record = find(from_record.id)
-            record.copy!(from_record)
-
-            record
+            update!(from_record)
           else
             create!(from_record)
           end
         end
 
+        # Public: Update an existing record of this record type with the
+        # attributes from the passed record. Only applies changes if from_record
+        # has been more recently updated than the last record synchronization.
+        #
+        # from_record - A Restforce::DB::Instances::Base instance.
+        #
+        # Returns a Restforce::DB::Instances::Base instance.
+        # Raises on any validation or external error.
+        def update!(from_record)
+          record = find(from_record.id)
+          return record if from_record.last_update < record.last_synchronize
+
+          record.copy!(from_record)
+        end
+
       end
 
     end

data/lib/restforce/db/record_types/salesforce.rb

@@ -19,7 +19,8 @@ module Restforce
         def create!(from_record)
           attributes = @mapping.convert(:salesforce, from_record.attributes)
           record_id = DB.client.create!(@record_type, attributes)
-          from_record.update!(salesforce_id: record_id)
+
+          from_record.update!(salesforce_id: record_id).after_sync
 
           find(record_id)
         end

data/lib/restforce/db/synchronizer.rb

@@ -19,7 +19,7 @@ module Restforce
       # last_run_time          - A Time object reflecting the time of the most
       #                          recent synchronization run. Runs will only
       #                          synchronize data more recent than this stamp.
-      def initialize(database_record_type, salesforce_record_type, last_run_time = nil)
+      def initialize(database_record_type, salesforce_record_type, last_run_time = DB.last_run)
         @database_record_type = database_record_type
         @salesforce_record_type = salesforce_record_type
         @last_run = last_run_time

data/lib/restforce/db/tracker.rb

@@ -0,0 +1,44 @@
+module Restforce
+
+  module DB
+
+    # Restforce::DB::Tracker encapsulates a minimal API to track and configure
+    # synchronization runtimes. It allows Restforce::DB to persist a "last
+    # successful sync" timestamp.
+    class Tracker
+
+      attr_reader :last_run
+
+      # Public: Initialize a Restforce::DB::Tracker. Sets a last_run timestamp
+      # on Restforce::DB if the supplied tracking file already contains a stamp.
+      #
+      # file_path - The Path to the tracking file.
+      def initialize(file_path)
+        @file = File.open(file_path, "a+")
+
+        timestamp = @file.read
+        return if timestamp.empty?
+
+        @last_run = Time.parse(timestamp)
+        Restforce::DB.last_run = @last_run
+      end
+
+      # Public: Persist the passed time in the tracker file.
+      #
+      # time - A Time object.
+      #
+      # Returns nothing.
+      def track(time)
+        @last_run = time
+
+        @file.truncate(0)
+        @file.print(time.utc.iso8601)
+        @file.flush
+        @file.rewind
+      end
+
+    end
+
+  end
+
+end

data/lib/restforce/db/version.rb

@@ -3,7 +3,7 @@ module Restforce
   # :nodoc:
   module DB
 
-    VERSION = "0.1.6"
+    VERSION = "0.2.3"
 
   end
 

data/lib/restforce/db/worker.rb

@@ -10,8 +10,6 @@ module Restforce
 
       class << self
 
-        attr_accessor :logger, :interval
-
         # Public: Store the list of currently open file descriptors so that they
         # may be reopened when a new process is spawned.
         #
@@ -41,6 +39,8 @@ module Restforce
 
       end
 
+      attr_accessor :logger, :tracker
+
       # Public: Initialize a new Restforce::DB::Worker.
       #
       # options - A Hash of options to configure the worker's run. Currently
@@ -50,7 +50,7 @@ module Restforce
       #           verbose  - Display command line output? Defaults to false.
       def initialize(options = {})
         @verbose = options.fetch(:verbose) { false }
-        self.class.interval = options.fetch(:interval) { DEFAULT_INTERVAL }
+        @interval = options.fetch(:interval) { DEFAULT_INTERVAL }
 
         Restforce::DB.configure { |config| config.parse(options[:config]) }
       end
@@ -72,16 +72,10 @@ module Restforce
         end
 
         loop do
-          log "=============================="
-
-          runtime = Benchmark.realtime do
-            Restforce::DB::RecordType.each do |name, record_type|
-              synchronize name, record_type
-            end
-          end
+          runtime = Benchmark.realtime { perform }
 
-          if runtime < self.class.interval && !stop?
-            sleep(self.class.interval - runtime)
+          if runtime < @interval && !stop?
+            sleep(@interval - runtime)
           end
 
           break if stop?
@@ -98,6 +92,43 @@ module Restforce
 
       private
 
+      # Internal: Perform the synchronization loop, recording the time that the
+      # run is performed so that future runs can pick up where the last run
+      # left off.
+      #
+      # Returns nothing.
+      def perform
+        track do
+          Restforce::DB::RecordType.each do |name, record_type|
+            synchronize name, record_type
+          end
+        end
+      end
+
+      # Internal: Run the passed block, updating the tracker with the time at
+      # which the run was initiated.
+      #
+      # Yields to a passed block.
+      # Returns nothing.
+      def track
+        if tracker
+          runtime = Time.now
+
+          if tracker.last_run
+            log "SYNCHRONIZING from #{tracker.last_run.iso8601}"
+          else
+            log "SYNCHRONIZING"
+          end
+
+          yield
+
+          log "DONE"
+          tracker.track(runtime)
+        else
+          yield
+        end
+      end
+
       # Internal: Synchronize the objects in the database and Salesforce
       # corresponding to the passed record type.
       #
@@ -106,9 +137,9 @@ module Restforce
       #
       # Returns a Boolean.
       def synchronize(name, record_type)
-        log "(#{name}) SYNCHRONIZING"
+        log "  SYNCHRONIZE #{name}"
         runtime =  Benchmark.realtime { record_type.synchronize }
-        log format("(#{name}) COMPLETED after %.4f", runtime)
+        log format("  COMPLETE after %.4f", runtime)
 
         return true
       rescue => e
@@ -133,8 +164,8 @@ module Restforce
       def log(text, level = :info)
         puts text if @verbose
 
-        return unless self.class.logger
-        self.class.logger.send(level, text)
+        return unless logger
+        logger.send(level, text)
       end
 
       # Internal: Log an error for the worker, outputting the entire error

data/lib/restforce/db.rb

@@ -16,8 +16,9 @@ require "restforce/db/record_types/salesforce"
 
 require "restforce/db/mapping"
 require "restforce/db/model"
-require "restforce/db/synchronizer"
 require "restforce/db/record_type"
+require "restforce/db/synchronizer"
+require "restforce/db/tracker"
 require "restforce/db/worker"
 
 module Restforce
@@ -28,6 +29,7 @@ module Restforce
 
     class << self
 
+      attr_accessor :last_run
       attr_writer :configuration
 
     end