jcangas-datagateway 1.1.0

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:major: 1
+:minor: 1
+:patch: 0

data/bin/datagw

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby -KU
+
+
+# == Sinopsis 
+#   DataGateway actua como una 'psarela de datos'. Permite trasnferir datos de 
+#	un origen a un destino, incluso en maquinas diferentes usando el proteocolo
+#   sftp.
+#   
+#
+# == Ejemplos
+#   
+#
+# == Usage
+#   Ejecute datagw.rb -g proyecto
+#
+#   Para ayuda use: datagw -h
+#
+# == Options
+#   -h, --help          Displays help message
+#   -e, --environment   Select the execution environment
+#   -q, --quiet         Output as little as possible, overrides verbose
+#   -v, --version       Print version and quit
+#   -d, --debug         Debug mode
+#
+# == Author
+#   Jorge L. Cangas
+#
+# == Copyright
+#   Copyright (c) 2008 Jorge L. Cangas
+
+require 'rubygems'
+require 'datagateway'
+
+AppConfig.run
+

data/lib/datagateway/conduits/csv_conduit.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require "faster_csv"
+
+class CSVConduit
+
+  def self.pack(data)
+    headers = data[0].attribute_names
+    headers.to_csv + data.map{ |rec|
+      headers.map{ |header| rec.send(header) }.to_csv(:force_quotes => true)
+    }.join
+  end
+
+  def self.unpack(data)
+    rows = FasterCSV.parse(data)
+    attribs = rows.delete_at(0)
+    attribs = attribs.map{|atr| atr.to_sym}
+    records = rows.map{|row| Hash[*attribs.zip(row).flatten]}
+  end
+end

data/lib/datagateway/conduits/yaml_conduit.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+class YAMLConduit
+
+  def self.pack(data)
+    YAML.dump(data.map(&:attributes))
+  end
+
+end

data/lib/datagateway/data_gateway.rb

@@ -0,0 +1,582 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'benchmark'
+require 'fileutils'
+require 'zip/zip'
+require 'folder_writer'
+require 'iconv'
+require 'ssh_transfer'
+require 'jcode'
+
+module DataGateway
+	CONDUIT_DIR = File.join(File.dirname(__FILE__), 'conduits')
+	@@resource_path = ''
+	@@use_resources = false
+	class << self
+		attr_accessor :logger, :inbox, :outbox, :donebox
+
+		def use_resources
+			@@use_resources			
+		end 		
+
+		def use_resources=(value)
+			@@use_resources = value
+		end 		
+
+		def conduit_for(formatid)
+			conduitid = File.basename(formatid.to_s, '.zip').split('.').last
+			conduit_class = conduitid.upcase + 'Conduit'
+			unless Object.const_defined?(conduit_class)			
+				conduit_file = conduitid.downcase + '_' + 'conduit.rb'
+				require  File.join(CONDUIT_DIR, conduit_file)
+			end
+			Object.const_get(conduit_class)
+		end
+
+		# Retrieve ActiveRecord class for a class name
+		def model_for(class_name)
+			return nil if class_name.nil?
+			# define a new ActiveRecord descendant if not found
+			class_name = class_name.to_sym
+			unless Object.const_defined?(class_name)
+				Object.const_set(class_name, Class.new(ActiveRecord::Base))
+			end
+			# Retrieve it
+			Object.const_get(class_name)
+		end
+	end
+	
+	class NullEncoder
+		def pack(data)
+			data
+		end
+	
+		def unpack(data)
+			data
+		end
+	end
+	
+	class Encoder
+		def initialize(options)
+			@pack = Iconv.new(options[:to], options[:from])
+			@unpack = Iconv.new(options[:from], options[:to])
+		end
+	
+		def pack(data)
+			@pack.iconv(data)
+		end
+	
+		def unpack(data)
+			@unpack.iconv(data)    
+		end
+	end
+	
+	class ColumnMapper
+		attr :options
+		def initialize(table_mapper, options = {})
+			@table_mapper = table_mapper
+			@options = options
+		end
+
+		def apply_to(record, value)
+			DataGateway.logger.debug "apply_to(#{record.inspect}, #{value}) options #{@options.inspect}"			
+			send(@options[:method], record, value)
+		end
+
+		# value mappers
+		def none(record, value)
+			# do nothing
+		end
+
+		def write_to(record, value)
+			record[@options[:write_to]] = value
+		end
+
+		def lookup(record, value)
+			
+			result_status = {}
+			new_value = lookup_map(value, result_status)
+			if result_status[:found]
+				write_to(record, new_value)
+			else
+				@table_mapper.register_fixup(value, self)
+			end
+		end
+
+	private
+		
+		def	lookup_map(value, result_status)
+			DataGateway.logger.debug "lookup_map(#{value}, #{result_status}) options #{@options.inspect}"			
+			lookup_model = DataGateway.model_for(@options[:table])
+			finder = "find_or_create_by_#{@options[:key]}"
+			lookup_rec = lookup_model.send(finder, value)
+			result_status[:found] = !(lookup_rec.nil?)
+			if result_status[:found]
+				lookup_rec.send(@options[:value]).to_s # valor mapeado
+			else
+				nil
+			end
+		end
+
+		def solve_fixup(record, value)
+			DataGateway.logger.debug "mapper #{@map_method} opt. #{@options.inspect}"
+			result_status = {}
+			DataGateway.logger.debug "lookup #{record.inspect} -> <#{value}>"
+			value = lookup_map(value, result_status)
+			if result_status[:found]
+				DataGateway.logger.debug "FOUND: #{value}"
+				write_to(record, value)
+			else
+				DataGateway.logger.debug "NOT FOUND!"
+			end
+		end
+	end
+
+	class TableMapper
+		attr :source
+		attr :identity
+		
+		def initialize(job, table)
+			@job = job
+			@source = table
+			@target = table
+			@column_map = {}
+			@identity = [:id]
+			@ar_fixups = []
+			@force_case = false
+			clear_fixups
+		end
+
+		def ignore!
+			@target = nil
+		end
+
+		def ignore?
+			@target == nil
+		end
+
+		def target(table_name = nil)
+			@target = table_name if table_name
+			@target
+		end
+		
+		def force_downcase
+			@force_case = :downcase
+		end
+		
+		def force_upcase
+			@force_case = :upcase
+		end
+
+		def target_model
+			DataGateway.model_for(@target)
+		end
+
+		# DSL syntax sugar for declare column mappers
+		def columns(column_map)
+			column_map.each_pair do |key, value|
+				case value
+					when Symbol, String: column_map[key] = write_to(value.to_sym)
+				end
+			end
+			@column_map.merge! column_map
+			
+			@column_map.each_pair do |key, value|
+				defaults = column_mapper_defaults(key)
+				@column_map[key].options.replace(defaults.merge!(@column_map[key].options))
+				#puts "map for #{key} => #{@column_map[key].options.inspect}"
+			end
+			@column_map
+		end
+
+		def none
+			create_column_mapper({:method => :none})
+		end
+
+		def write_to(column)
+			create_column_mapper({:method => :write_to, :write_to => column})
+		end
+
+		def lookup(options)
+			options[:method] = :lookup
+			create_column_mapper(options)
+		end
+
+		def identity_by(*cols)
+			@identity = cols
+		end
+		
+	 # utilities
+		def column_mapper_for(column_name)
+			@column_map[column_name] ||= create_column_mapper(column_mapper_defaults(column_name))
+		end
+		
+		def column_mapper_defaults(column_name)
+			column_name = column_name.to_s.send(@force_case).to_sym if @force_case
+			case column_name.to_s
+				when /(.*)_id$/i:	{	:method => :lookup,
+														:write_to => column_name,
+														:table => $1.downcase.camelize.to_sym,
+														:key => @job.get_mapper($1.downcase.camelize.to_sym).identity,
+														:value => :id
+													}
+			else 	{	:method => :write_to,
+							:write_to => column_name
+						}
+			end
+			
+		end		
+
+		def update_all(records)
+			DataGateway.logger.debug "importing #{records.size} recods into table #{source}"
+			klass = target_model
+			DataGateway.logger.debug "target class #{klass} (PK: #{klass.primary_key})"
+			records.each { |rec|
+				rec = map(rec)
+				update_rec(rec) unless current_fixups(rec)
+			}
+		end
+
+		def clear_fixups
+			@fixups = {}
+		end
+
+		def register_fixup(value, col_mapper)
+			@ar_fixups << [value, col_mapper]
+		end
+
+		def current_fixups(rec)
+			DataGateway.logger.debug "current fixups for: #{rec.inspect}"
+			DataGateway.logger.debug @ar_fixups.inspect
+			return false if @ar_fixups.empty?
+			@fixups[rec] ||= []
+			@fixups[rec].concat @ar_fixups
+			@ar_fixups = []
+			true
+		end
+
+		def solve_fixups
+			DataGateway.logger.debug "start solve_fixups #{@fixups.size}"
+			@fixups.each_pair { |record, data|
+				data.each{|item|
+					value = item[0]
+					col_mapper = item[1]
+					col_mapper.solve_fixup(record, value)
+				}
+				update_rec(record)
+			}
+			DataGateway.logger.debug "end solve_fixups"
+		ensure
+			clear_fixups
+		end
+
+		def update_rec(rec)
+			DataGateway.logger.debug "update rec class #{target_model} '->' #{rec.inspect}"
+			conditions = identity.map{|key| key.to_s}
+			values = conditions.map{|key| rec.delete(key.to_sym)}
+			finder = 'find_or_create_by_' + conditions.join('_')
+			DataGateway.logger.debug "finder: #{finder}(#{values.join(',')})"
+		
+			ar = target_model.send(finder, *values)
+			DataGateway.logger.debug "get: #{ar.inspect}"
+			DataGateway.logger.debug "upd atrt: #{rec.inspect}"
+			ar.update_attributes(rec)
+			return ar[:id]
+		end
+
+		def map(ar)
+			new_ar = {}
+			ar.each_pair do |key, value|
+				column_mapper_for(key).apply_to(new_ar, value)
+			end
+			new_ar
+		end
+	private
+		def create_column_mapper( options)
+			ColumnMapper.new(self, options)
+		end
+	end
+
+	class Job
+		attr_accessor :format_id
+		attr :transfer
+		attr :name
+		
+		def initialize(name, &block)
+			@name = name
+			DataGateway.logger.info "DataGateway.use_resources #{DataGateway.use_resources}"
+			@use_resources = DataGateway.use_resources
+			DataGateway.logger.info "resources will be imported!" if @use_resources
+			@encoder = NullEncoder.new
+			@transfer = nil
+			instance_eval(&block) if block_given?
+		end
+		
+		def transfer_for(options)		
+			options.merge! :logger => DataGateway.logger
+			@transfer = SSHTransfer.new(options)
+		end
+		
+		def run
+			AppConfig.logger.info "running job #{name}"
+			do_run
+			AppConfig.logger.info "job #{name} finished"
+		end
+
+	private	
+		# only for debug purpouse
+		def use_resources(bool)
+			@use_resources = bool
+		end
+		
+		def encode(options)
+			@encoder = Encoder.new(options)
+		end
+		
+		def pack(data)
+			DataGateway.conduit_for(self.format_id).pack(data)
+		end
+
+		def unpack(data)
+			DataGateway.conduit_for(format_id).unpack(data)
+		end
+
+		def dbchange_point
+			@dbchange_point ||= DBChangePoint.class_for(ActiveRecord::Base.connection).new
+		end
+	end
+	
+	class Import < Job
+		def initialize(name, &block)
+			@mappers = {} #import mappers
+			super
+		end
+
+		def do_run
+			transfer.download if transfer
+
+			Dir.glob(File.join(DataGateway::inbox, '*.zip')).sort.each do |filename|
+			DataGateway.logger.info "Importing #{filename}"
+			import_file(filename)
+			FileUtils.mv(filename, DataGateway::donebox)
+			DataGateway.logger.info "Import done for #{filename}"
+			end		
+		end
+
+		def get_mapper(table)
+			@mappers[table] ||= TableMapper.new(self, table)
+		end
+		
+	private # DSL
+
+		def download_from(options)
+			options.merge! 'local_files' => DataGateway.inbox
+			transfer_for(options)
+		end
+
+		def importing(table, &block)
+			get_mapper(table).instance_eval(&block) if block_given?
+		end
+
+	private
+
+		def import_file(file_name)
+			self.format_id = file_name
+			clear_fixups
+			begin
+				Zip::ZipInputStream::open(file_name) { |io|
+					while (entry = io.get_next_entry)
+						content = io.read
+						fname = entry.name
+						if fname =~ /^attach\//
+							fname = fname.gsub(/^attach\//, '')
+							import_attach(fname, content)
+						else
+							class_name = File.basename(fname, '.*').downcase.camelize.to_sym
+							import_class(class_name, content)
+						end
+					end
+				}
+			ensure
+				solve_fixups
+			end
+		end
+
+		def clear_fixups
+			@mappers.each_value {|mapper|
+				mapper.clear_fixups
+			}
+		end
+
+		def solve_fixups
+			@mappers.each_value {|mapper|
+				mapper.solve_fixups
+			}
+		end
+
+		def import_class(class_name, data)
+			DataGateway.logger.info "Importing data for #{class_name}"
+			mapper = get_mapper(class_name)
+			if mapper.ignore?
+				DataGateway.logger.info "Ignored"
+				return
+			end
+			mapper.update_all unpack(data)
+			#p dbchange_point.capture_for(klass)
+			DataGateway.logger.info "done for #{class_name}"
+		end
+
+		def import_attach(fname, content)
+			return unless @use_resources
+			DataGateway.logger.info "importing attach #{fname}"
+			target_file = File.expand_path(fname, DataGateway.resource_path)
+			FileUtils.mkdir_p(File.dirname(target_file))
+			File.open(target_file, "wb") {|f| f.write content}
+		end
+	end
+	
+	class Export < Job
+		def initialize(name, &block)
+			@exports = {}
+			@exports_order = []
+			export_to :csv #default format
+			super		
+		end
+		
+		def each
+			tables = @exports_order
+			tables.each { |cname|
+			  yield cname, export_class(cname)
+			}
+		end
+		
+		def export_folder
+			@export_folder ||= create_folder
+		end
+		
+		def close_export_folder
+			@export_folder.close if @export_folder
+		end
+		
+		def do_run
+			DataGateway.logger.debug 'resource_path ' + DataGateway.resource_path
+			begin				
+				self.each { |cname, records|
+					file_name = "#{cname}.#{self.format_id}"
+					DataGateway.logger.debug "Exporting archive #{file_name}"
+					#records.each { |rec|
+					#		rec.each {|key, value|
+					#				rec[key] = @encoder.pack(rec[key]) unless (key =~ /^img/i)
+					#		}
+					#}
+					write_file(file_name, records)
+				}
+			ensure
+				close_export_folder
+			end
+			
+			transfer.upload if transfer				
+		end
+	
+	private # DSL
+		def upload_to(options)
+			options.merge! 'local_files' => File.join(DataGateway.outbox, "*.zip")
+			transfer_for(options)
+		end
+		
+		def exporting(table_name, options = {})
+			@exports_order << table_name
+			@exports[table_name] = options
+		end
+
+		def export_to(format)
+			self.format_id = format
+		end
+  private
+		def create_folder(outbox = DataGateway::outbox)
+			FileUtils.mkpath(outbox)
+			stamp = Time.now.strftime("%Y%m%d%H%M%S")
+			folder_name = File.join(outbox, "#{stamp}.#{self.format_id}")
+			(APP_ENV == 'development') ? FolderWriter.new(folder_name) :
+										ZipFolderWriter.new(folder_name)
+		end
+
+		def export_class(cname)
+			result = []
+			bm = Benchmark.measure {
+				klass = DataGateway.model_for(cname.to_s.downcase.camelize)
+				DataGateway.logger.info "Exporting #{klass} (#{self.format_id})"
+				records = klass.find(:all, @exports[cname])
+				records.each { |r| export_attachments(r) } if @use_resources
+				result = pack(records) unless records.empty?
+			}
+			DataGateway.logger.info "data exported in #{bm}"
+			result
+		end
+
+		def export_attachments(record)			
+			record.attributes.keys.select { |key| key =~ /^img/i }.each { |key|
+				file = record.attributes[key]
+				#TODO Si el path del attach es absoluto, quiza debemos tener cuidado al descomprimir el zip
+				next if file.blank?
+				path = File.expand_path(file, DataGateway.resource_path)
+				DataGateway.logger.info "exporting attach #{path}"
+				#TODO Si el fichero ya se exporto solo en este vuelta, no repetir!
+				next unless File.exist?(path)
+				attach_data = File.open(path, 'rb') { |f| f.read }
+				write_file(File.join('attach', file), attach_data, 'b')
+			}
+		end
+
+		def write_file(file_name, data, options = '')
+			op = StringIO.new("", "w")
+			op.puts data
+			if (options == 'b')
+				export_folder.open(file_name, "w" + options){|f| f.puts op.string}				
+			else
+				export_folder.open(file_name, "w" + options){|f| f.puts @encoder.pack(op.string)}
+			end
+		end
+	end
+
+	# DataGateway DSL	
+	def resource_path(path= nil)
+		if path
+			@@resource_path = File.expand_path(path)
+		else
+			@@resource_path || ''
+		end
+	end
+	
+	def db_connection(conn)
+		ActiveRecord::Base.establish_connection conn
+		if conn['nax_empresa']
+			NAX::ActiveRecord::Base.establish_connection conn if NAX::ActiveRecord::Base.current_connection.nil?
+		end
+	end
+
+	def nax_connection(connection)
+		NAX::ActiveRecord::Base.establish_connection connection
+	end		
+
+	def import(name, &block)
+		name = name.to_sym
+		self.class.class_eval {
+			attr_reader name
+		}
+		new_job = Import.new(name, &block)
+		instance_variable_set('@' + name.to_s, new_job)
+		return  new_job
+	end
+
+	def export(name, &block)
+		name = name.to_sym
+		self.class.class_eval {
+			attr_reader name
+		}
+		new_job = Export.new(name, &block)
+		instance_variable_set('@' + name.to_s, new_job)
+		return  new_job
+	end
+end

data/lib/datagateway/dbchange_point.rb

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+	class DBChangePoint #memento
+		def self.class_for(connection)
+			cname = connection.class.name.split('::').last
+			cname = cname.chomp('Adapter') + 'ChangePoint'
+			Object.const_get(cname)
+		end
+	end
+
+	class SQLServerChangePoint #memento
+		def initialize
+			@change_point = {}
+		end
+
+		def capture_for(arclass)
+			@change_point[arclass.name] = arclass.connection.select_value("select @@dbts")
+		end
+
+		def changed_point_for(arclass)
+			@change_point[arclass.name] ||= [0,0,0,0] #binary(8) in MSSQLServer
+		end
+
+		def changed_condition_for(arclass)
+			['DBREV > ?', @change_point[arclass.name].to_int]
+		end
+	end