seven1m-onebody-updateagent 0.1.0

data/README.markdown

@@ -0,0 +1,77 @@
+OneBody UpdateAgent
+===================
+
+Ruby gem that pushes data from a membership data source to a remote OneBody instance via the REST API.
+
+Download and Install
+--------------------
+
+Run the following command ("sudo" may be required in some environments):
+
+    gem install seven1m-onebody-updateagent -s http://gems.github.com
+    
+Configuration
+-------------
+
+1. Run `update_onebody` at a terminal, then take note of where the example.yml config file resides.
+2. Copy the example config file to a convenient location and edit appropriately.
+
+Your "site" address will probably be something like "http://example.com" or
+"http://yoursite.beonebody.com".
+
+You can get your user api key from OneBody (you must be a super user) by running the following
+command (on the server):
+
+    cd /path/to/onebody
+    rake onebody:api:key EMAIL=admin@example.com
+    
+If your instance is hosted at <http://beonebody.com>, you may email <support@beonebody.com>.
+
+Preparation
+-----------
+
+Using your membership management software, reporting solution, database utility, custom script, etc.,
+export your people and family data to a single comma separated values (CSV) file, e.g. people.csv.
+
+Duplicate family data should be present for each member of the same family.
+
+The first row of the file is the attribute headings, and must match the attributes available:
+* http://github.com/seven1m/onebody/tree/master/app/models/person.rb
+* http://github.com/seven1m/onebody/tree/master/app/models/family.rb (prefix each with "family_")
+
+Not all attributes are required. 
+
+Optionally, if you have shell access to your hosted instance of OneBody, you can run
+`rake onebody:export:people:csv` to export the current OneBody data (if you have any records in
+the OneBody database) as a starting point.
+
+Use the following attributes to track the identity/foreign keys from your existing membership
+management database. Do *not* include "id" and "family_id" columns.
+* legacy\_id
+* legacy\_family\_id
+
+Converters
+----------
+
+As of this writing, one Church Management System (ChMS) is supported via a Converter. The
+Converter translates field names and data into the formats and locations expected by OneBody.
+
+To use a converter, you must specify it in your config file. The example config file has these
+settings disabled; simply remove the pound sign at the beginning of each line to enable the
+use of the converter. Specify the name and any additional settings.
+
+Usage
+-----
+
+To run UpdateAgent:
+
+    update_onebody -c path/to/config.yml path/to/people.yml
+    
+If you plan to scheduled UpdateAgent to run periodically without human intervention,
+you'll want to at least use the `-y` switch, which assumes *yes* to any questions:
+
+    update_onebody -c path/to/config.yml -y path/to/people.yml
+    
+You may also log all output:
+
+    update_onebody -c path/to/config.yml -y -l path/to/updateagent.log path/to/people.yml

data/bin/update_onebody

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'yaml'
+
+options = {:confirm => true, :force => false}
+opt_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: ruby updateagent.rb -c path/to/config.yml [options] path/to/people.csv"
+  opts.on("-y", "--no-confirm", "Assume 'yes' to any questions") do |v|
+    options[:confirm] = false
+  end
+  opts.on("-l", "--log LOGFILE", "Output to log rather than stdout") do |log|
+    $stdout = $stderr = File.open(log, 'a')
+  end
+  opts.on("-f", "--force", "Force update all records") do |f|
+    options[:force] = true
+  end
+  opts.on("-c", "--config-file PATH", "Path to configuration file") do |c|
+    options[:config_file] = c
+    config = YAML::load(File.read(c))
+    SITE       = config['site']
+    USER_EMAIL = config['user_email']
+    USER_KEY   = config['user_key']
+    if c = config['converter']
+      USE_CONVERTER = c['name']
+      CONVERTER_CONFIG = c
+    end
+  end
+end
+opt_parser.parse!
+
+unless options[:config_file]
+  puts opt_parser.help
+  puts
+  puts 'You must specify a config file containing site and login info.'
+  puts "See #{File.expand_path(File.dirname(File.dirname(__FILE__)))}/example.yml"
+  puts
+  exit
+end
+
+if ARGV[0] # path/to/people.csv
+  require 'onebody-updateagent'
+  puts "Update Agent running at #{Time.now.strftime('%m/%d/%Y %I:%M %p')}"
+  agent = PeopleUpdater.new(ARGV[0])
+  puts "comparing records..."
+  agent.compare(options[:force])
+  if agent.has_work?
+    if options[:confirm]
+      case ask("#{agent.create.length + agent.update.length} record(s) to push. Continue? (Yes, No, Review) ") { |q| q.in = %w(yes no review y n r) }
+      when 'review', 'r'
+        agent.present
+        unless agent.confirm
+          puts "Canceled by user\n"
+          exit
+        end
+      when 'no', 'n'
+        puts "Canceled by user\n"
+        exit
+      end
+    end
+    agent.push
+    puts "Completed at #{Time.now.strftime('%m/%d/%Y %I:%M %p')}\n\n"
+  else
+    puts "Nothing to push\n\n"
+  end
+else
+  puts opt_parser.help
+end

data/example.yml

@@ -0,0 +1,20 @@
+site:       http://localhost:3000
+user_email: admin@example.com
+user_key:   dafH2KIiAcnLEr5JxjmX2oveuczq0R6u7Ijd329DtjatgdYcKp
+#converter:
+#  name: ACS
+#  visible:
+#    - Member
+#    - Visitor
+#    - Prospect
+#    - Attender
+#    - College
+#    - Other Child
+#    - Other Youth
+#  visible_on_printed_directory:
+#    - Member
+#    - Attender
+#  full_access:
+#    - Member
+#    - Attender
+#    - College

data/example_using_converter.yml

@@ -0,0 +1,20 @@
+site:       http://localhost:3000
+user_email: admin@example.com
+user_key:   dafH2KIiAcnLEr5JxjmX2oveuczq0R6u7Ijd329DtjatgdYcKp
+converter:
+  name: ACS
+  visible:
+    - Member
+    - Visitor
+    - Prospect
+    - Attender
+    - College
+    - Other Child
+    - Other Youth
+  visible_on_printed_directory:
+    - Member
+    - Attender
+  full_access:
+    - Member
+    - Attender
+    - College

data/lib/onebody-updateagent.rb

@@ -0,0 +1,19 @@
+require 'date'
+require 'rubygems'
+require 'fastercsv'
+require 'highline/import'
+require 'activeresource'
+require 'digest/sha1'
+
+HighLine.track_eof = false
+
+require 'updateagent/resources'
+require 'updateagent/schema'
+require 'updateagent/hash_extensions'
+require 'updateagent/updateagent'
+require 'updateagent/updaters/people_updater'
+require 'updateagent/updaters/family_updater'
+
+if defined?(USE_CONVERTER)
+  require "updateagent/converters/#{USE_CONVERTER.downcase}_converter"
+end

data/lib/updateagent/converters/acs_converter.rb

@@ -0,0 +1,84 @@
+class ACSConverter
+
+  def straight
+    {
+      "FamilyNumber"     => "legacy_family_id",
+      "IndividualNumber" => "sequence",
+      "LastName"         => "last_name",
+      "Suffix"           => "suffix",
+      "Address1"         => "family_address1",
+      "Address2"         => "family_address2",
+      "City"             => "family_city",
+      "State"            => "family_state",
+      "ZIPCode"          => "family_zip",
+      "DateOfBirth"      => "birthday",
+      "HomeEmailAddr"    => "email",
+      "FamilyName"       => "family_name",
+    }
+  end
+  
+  def convert(records)
+    create_family_names(records)
+    records.map do |record|
+      convert_record(record)
+    end
+  end
+  
+  def convert_record(record)
+    returning({}) do |new_record|
+      record.each do |key, value|
+        if new_key = straight[key]
+          new_record[new_key] = value
+        end
+      end
+      new_record['legacy_id'] = new_record['legacy_family_id'] + new_record['sequence']
+      new_record['first_name'] = get_first_name(record)
+      new_record['gender'] = record['FamilyPosition'] == 'Child' ? {'Male' => 'Boy', 'Female' => 'Girl'}[record['Gender']] : record['Gender']
+      new_record['family_home_phone'] = get_phone(record, 'Home')
+      new_record['work_phone'] = get_phone(record, 'Business')
+      new_record['mobile_phone'] = get_phone(record, 'Cell')
+      new_record['fax'] = get_phone(record, 'FAX')
+      new_record['family_last_name'] = new_record['last_name']
+      new_record['family_name'] = @family_names[new_record['legacy_family_id']]
+      new_record['visible_to_everyone'] = CONVERTER_CONFIG['visible'].include?(record['MemberStatus'])
+      new_record['visible_on_printed_directory'] = CONVERTER_CONFIG['visible_on_printed_directory'].include?(record['MemberStatus'])
+      new_record['full_access'] = CONVERTER_CONFIG['full_access'].include?(record['MemberStatus'])
+      new_record['email'] = record['HomeEmailAddr'].to_s.any? ? record['HomeEmailAddr'] : record['BusinessEmailAddr']
+    end
+  end
+  
+  def create_family_names(records)
+    # this could probably be less messy
+    @families = {}
+    records.each do |record|
+      @families[record['FamilyNumber']] ||= {}
+      @families[record['FamilyNumber']][record['FamilyPosition']] = record
+    end
+    @family_names = {}
+    @families.each do |family_number, family|
+      family['Head'] ||= family.delete('Spouse') || family.delete('Child')
+      if family['Head']
+        if family['Spouse']
+          @family_names[family_number] = "#{get_first_name(family['Head'])} & #{get_first_name(family['Spouse'])} #{family['Head']['LastName']}"
+        else
+          @family_names[family_number] = "#{get_first_name(family['Head'])} #{family['Head']['LastName']}"
+        end
+      else
+        @family_names[family_number] = "#{get_first_name(family['Other'])} #{family['Other']['LastName']}"
+      end
+    end
+  end
+  
+  def get_first_name(record)
+    record['GoesByName'].to_s.any? ? record['GoesByName'] : record['FirstName']
+  end
+  
+  def get_phone(record, type)
+    unless record[type + 'Unlisted'].to_s.downcase == 'true'
+      phone = record[type + 'Phone'].to_s.scan(/\d/).join
+      phone << ' ' + record[type + 'Extension'] if record[type + 'Extension'].to_s.any?
+      phone
+    end
+  end
+
+end

data/lib/updateagent/hash_extensions.rb

@@ -0,0 +1,21 @@
+class Hash
+  # creates a uniq sha1 digest of the hash's values
+  # should mirror similar code in OneBody's lib/db_tools.rb
+  def values_hash(*attrs)
+    attrs = keys.sort unless attrs.any?
+    attrs = attrs.first if attrs.first.is_a?(Array)
+    values = attrs.map do |attr|
+      value = self[attr.to_s]
+      if value.respond_to?(:strftime)
+        value.strftime('%Y-%m-%d %H:%M:%S')
+      elsif value == true
+        1
+      elsif value == false
+        0
+      else
+        value
+      end
+    end
+    DEBUG ? values.join : Digest::SHA1.hexdigest(values.join)
+  end
+end

data/lib/updateagent/resources.rb

@@ -0,0 +1,8 @@
+class Base < ActiveResource::Base
+  self.site     = SITE
+  self.user     = USER_EMAIL
+  self.password = USER_KEY
+end
+
+class Person < Base; end
+class Family < Base; end

data/lib/updateagent/schema.rb

@@ -0,0 +1,15 @@
+class Schema
+  def initialize(resource)
+    @schema = resource.get(:schema)
+  end
+  def type(t)
+    @schema.select { |c| c['type'] == t }.map { |c| c['name'] }.uniq
+  end
+end
+person_schema = Schema.new(Person)
+family_schema = Schema.new(Family)
+
+DATETIME_ATTRIBUTES = person_schema.type(:datetime) + family_schema.type(:datetime).map { |c| 'family_' + c }
+BOOLEAN_ATTRIBUTES  = person_schema.type(:boolean)  + family_schema.type(:boolean).map  { |c| 'family_' + c }
+INTEGER_ATTRIBUTES  = person_schema.type(:integer)  + family_schema.type(:integer).map  { |c| 'family_' + c }
+IGNORE_ATTRIBUTES   = %w(updated_at created_at family_updated_at family_latitude family_longitude)

data/lib/updateagent/updateagent.rb

@@ -0,0 +1,160 @@
+MAX_HASHES_AT_A_TIME = 1000
+MAX_TO_BATCH_AT_A_TIME = 25
+
+DEBUG = false
+
+# general class to handle comparing and pushing data to the remote end
+class UpdateAgent
+  def initialize(data=nil)
+    @attributes = []
+    @data = []
+    @create = []
+    @update = []
+    if data
+      if data.is_a?(Array)
+        @data = data
+        @attributes = data.first.keys.sort
+      else
+        read_from_file(data)
+      end
+    end
+    if invalid = @data.detect { |row| row['id'] }
+      puts "Error: one or more records contain an 'id' column."
+      puts "You must utilize 'legacy_id' rather than 'id' so that"
+      puts "identity and foreign keys are maintained from your"
+      puts "existing membership management database."
+      exit
+    end
+  end
+  
+  # load data from csv file and do some type conversion for bools and dates
+  # first row must be attribute names
+  def read_from_file(filename)
+    csv = FasterCSV.open(filename, 'r')
+    @attributes = csv.shift
+    record_count = 0
+    @data = csv.map do |row|
+      hash = {}
+      row.each_with_index do |value, index|
+        key = @attributes[index]
+        next if IGNORE_ATTRIBUTES.include?(key)
+        if DATETIME_ATTRIBUTES.include?(key)
+          if value.blank?
+            value = nil
+          else
+            begin
+              value = DateTime.parse(value)
+            rescue ArgumentError
+              puts "Invalid date in #{filename} record #{index} (#{key}) - #{value}"
+              exit(1)
+            end
+          end
+        elsif BOOLEAN_ATTRIBUTES.include?(key)
+          if value == '' or value == nil
+            value = nil
+          elsif %w(no false 0).include?(value.downcase)
+            value = false
+          else
+            value = true
+          end
+        elsif INTEGER_ATTRIBUTES.include?(key)
+          value = value.to_s != '' ? value.scan(/\d/).join.to_i : nil
+        end
+        hash[key] = value
+      end
+      record_count += 1
+      print "reading record #{record_count}\r"
+      hash
+    end
+    puts
+    @attributes.reject! { |a| IGNORE_ATTRIBUTES.include?(a) }
+  end
+  
+  def ids
+    @data.map { |r| r['id'] }.compact
+  end
+  
+  def legacy_ids
+    @data.map { |r| r['legacy_id'] }.compact
+  end
+
+  def compare(force=false)
+    compare_hashes(legacy_ids, force)
+  end
+  
+  def has_work?
+    (@create + @update).any?
+  end
+
+  def present
+    puts "The following #{resource.name.downcase} records will be pushed..."
+    puts 'legacy id  name'
+    puts '---------- -------------------------------------'
+    @create.each { |r| present_record(r, true) }
+    @update.each { |r| present_record(r) }
+    puts
+  end
+  
+  def present_record(row, new=false)
+    puts "#{row['legacy_id'].to_s.ljust(10)} #{name_for(row).to_s.ljust(40)} #{new ? '(new)' : '     '}"
+    if DEBUG
+      puts row.values_hash(@attributes)
+      puts row['remote_hash']
+    end
+  end
+  
+  def confirm
+    agree('Do you want to continue, pushing these records to OneBody? ')
+  end
+  
+  # use ActiveResource to create/update records on remote end
+  def push
+    puts 'Updating remote end...'
+    index = 0
+    print "#{resource.name} 0/0\r"; STDOUT.flush
+    (@create + @update).each_slice(MAX_TO_BATCH_AT_A_TIME) do |records|
+      response = resource.post(:batch, {}, records.to_xml)
+      statuses = Hash.from_xml(response.body)['records']
+      statuses.select { |s| s['status'] == 'error' }.each do |status|
+        puts "#{status['legacy_id']}: #{status['error']}"
+      end
+      index += records.length
+      print "#{resource.name} #{index}/#{@create.length + @update.length}\r"; STDOUT.flush
+    end
+    puts
+  end
+  
+  def data_by_id
+    @data_by_id ||= begin
+      by_id = {}
+      @data.each { |r| by_id[r['legacy_id'].to_i] = r }
+      by_id
+    end
+  end
+
+  attr_accessor :attributes, :data
+  attr_reader :update, :create
+  
+  class << self; attr_accessor :resource; end
+  def resource; self.class.resource; end
+  
+  protected
+  
+  # ask remote end for value hashe for each record (50 at a time) 
+  # mark records to create or update based on response
+  def compare_hashes(ids, force=false)
+    ids.each_slice(MAX_HASHES_AT_A_TIME) do |some_ids|
+      print '.'; STDOUT.flush
+      options = {:attrs => @attributes.join(','), :legacy_id => some_ids.join(',')}
+      options.merge!(:debug => true) if DEBUG
+      hashes = resource.get(:hashify, options)
+      hashes.each do |record|
+        row = data_by_id[record['legacy_id'].to_i]
+        row['remote_hash'] = record['hash'] if DEBUG
+        @update << row if force or row.values_hash(@attributes) != record['hash']
+      end
+      @create += some_ids.reject { |id| hashes.map { |h| h['legacy_id'].to_i }.include?(id.to_i) }.map { |id| data_by_id[id.to_i] }
+    end
+    puts
+  end
+end

data/lib/updateagent/updaters/family_updater.rb

@@ -0,0 +1,6 @@
+class FamilyUpdater < UpdateAgent
+  self.resource = Family
+  def name_for(row)
+    row['name']
+  end
+end

data/lib/updateagent/updaters/people_updater.rb

@@ -0,0 +1,72 @@
+# handles people.csv and splits out family data for FamilyUpdater
+class PeopleUpdater < UpdateAgent
+  self.resource = Person
+  
+  # split out family data and create a new FamilyUpdater
+  def initialize(filename)
+    super(filename)
+    if defined?(USE_CONVERTER)
+      converter = Kernel.const_get(USE_CONVERTER + 'Converter').new
+      @data = converter.convert(@data.clone)
+      @attributes = @data.first.keys
+    end
+    person_data = []
+    family_data = {}
+    @data.each_with_index do |row, index|
+      person, family = split_change_hash(row)
+      if existing_family = family_data[family['legacy_id']]
+        person['family'] = existing_family
+        person_data << person
+      else
+        person['family'] = family
+        person_data << person
+        family_data[family['legacy_id']] = family
+      end
+      print "splitting family record #{index+1}\r"
+    end
+    puts
+    @data = person_data
+    @attributes.reject! { |a| a =~ /^family_/ and a != 'family_id' }
+    @family_agent = FamilyUpdater.new(family_data.values)
+  end
+  
+  def name_for(row)
+    "#{row['first_name']} #{row['last_name']}"
+  end
+  
+  def compare(force=false)
+    @family_agent.compare(force)
+    super(force)
+  end
+  
+  def has_work?
+    @family_agent.has_work? or super
+  end
+  
+  def present
+    @family_agent.present if @family_agent.has_work?
+    super
+  end
+  
+  def push
+    @family_agent.push
+    super
+  end
+  
+  protected
+  
+    # split hash of values into person and family values based on keys
+    def split_change_hash(vals)
+      person_vals = {}
+      family_vals = {}
+      vals.each do |key, val|
+        if key =~ /^family_/
+          family_vals[key.sub(/^family_/, '')] = val
+        else
+          person_vals[key] = val
+        end
+      end
+      family_vals['legacy_id'] ||= person_vals['legacy_family_id']
+      [person_vals, family_vals]
+    end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: seven1m-onebody-updateagent
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Tim Morgan
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-13 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: highline
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: activeresource
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: tim@timmorgan.org
+executables: 
+- update_onebody
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- example.yml
+- example_using_converter.yml
+- README.markdown
+- bin/update_onebody
+- lib/onebody-updateagent.rb
+- lib/updateagent/hash_extensions.rb
+- lib/updateagent/resources.rb
+- lib/updateagent/schema.rb
+- lib/updateagent/updateagent.rb
+- lib/updateagent/converters/acs_converter.rb
+- lib/updateagent/updaters/family_updater.rb
+- lib/updateagent/updaters/people_updater.rb
+has_rdoc: false
+homepage: http://github.com/seven1m/onebody-updateagent
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Companion to OneBody that handles sync with external data source.
+test_files: []
+