jcangas-datagateway 1.2.2 → 1.4.4

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 2
+:patch: 4
 :major: 1
-:minor: 2
+:minor: 4

data/lib/datagateway/data_gateway.rb

@@ -1,33 +1,36 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
+require 'jcode'
 require 'benchmark'
 require 'fileutils'
 require 'zip/zip'
 require 'folder_writer'
 require 'iconv'
 require 'ssh_transfer'
-require 'jcode'
+require 'job'
+require 'active_support'
+
 
 module DataGateway
 	CONDUIT_DIR = File.join(File.dirname(__FILE__), 'conduits')
 	@@resource_path = ''
 	@@use_resources = false
 	class << self
-		attr_accessor :logger, :inbox, :outbox, :donebox
+		attr_accessor :last_time, :logger, :inbox, :outbox, :donebox
 
 		def use_resources
-			@@use_resources			
-		end 		
+			@@use_resources
+		end
 
 		def use_resources=(value)
 			@@use_resources = value
-		end 		
+		end
 
 		def conduit_for(formatid)
 			conduitid = File.basename(formatid.to_s, '.zip').split('.').last
 			conduit_class = conduitid.upcase + 'Conduit'
-			unless Object.const_defined?(conduit_class)			
+			unless Object.const_defined?(conduit_class)
 				conduit_file = conduitid.downcase + '_' + 'conduit.rb'
 				require  File.join(CONDUIT_DIR, conduit_file)
 			end
@@ -46,498 +49,10 @@ module DataGateway
 			Object.const_get(class_name)
 		end
 	end
-	
-	class NullEncoder
-		def pack(data)
-			data
-		end
-	
-		def unpack(data)
-			data
-		end
-	end
-	
-	class Encoder
-		def initialize(options)
-			@pack = Iconv.new(options[:to], options[:from])
-			@unpack = Iconv.new(options[:from], options[:to])
-		end
-	
-		def pack(data)
-			@pack.iconv(data)
-		end
-	
-		def unpack(data)
-			@unpack.iconv(data)    
-		end
-	end
-	
-	class ColumnMapper
-		attr :options
-		def initialize(table_mapper, options = {})
-			@table_mapper = table_mapper
-			@options = options
-		end
-
-		def apply_to(record, value)
-			DataGateway.logger.debug "apply_to(#{record.inspect}, #{value}) options #{@options.inspect}"			
-			send(@options[:method], record, value)
-		end
-
-		# value mappers
-		def none(record, value)
-			# do nothing
-		end
-
-		def write_to(record, value)
-			record[@options[:write_to]] = value
-		end
-
-		def lookup(record, value)
-			
-			result_status = {}
-			new_value = lookup_map(value, result_status)
-			if result_status[:found]
-				write_to(record, new_value)
-			else
-				@table_mapper.register_fixup(value, self)
-			end
-		end
-
-	private
-		
-		def	lookup_map(value, result_status)
-			DataGateway.logger.debug "lookup_map(#{value}, #{result_status}) options #{@options.inspect}"			
-			lookup_model = DataGateway.model_for(@options[:table])
-			finder = "find_or_create_by_#{@options[:key]}"
-			lookup_rec = lookup_model.send(finder, value)
-			result_status[:found] = !(lookup_rec.nil?)
-			if result_status[:found]
-				lookup_rec.send(@options[:value]).to_s # valor mapeado
-			else
-				nil
-			end
-		end
-
-		def solve_fixup(record, value)
-			DataGateway.logger.debug "mapper #{@map_method} opt. #{@options.inspect}"
-			result_status = {}
-			DataGateway.logger.debug "lookup #{record.inspect} -> <#{value}>"
-			value = lookup_map(value, result_status)
-			if result_status[:found]
-				DataGateway.logger.debug "FOUND: #{value}"
-				write_to(record, value)
-			else
-				DataGateway.logger.debug "NOT FOUND!"
-			end
-		end
-	end
-
-	class TableMapper
-		attr :source
-		attr :identity
-		
-		def initialize(job, table)
-			@job = job
-			@source = table
-			@target = table
-			@column_map = {}
-			@identity = [:id]
-			@ar_fixups = []
-			@force_case = false
-			clear_fixups
-		end
-
-		def ignore!
-			@target = nil
-		end
-
-		def ignore?
-			@target == nil
-		end
-
-		def target(table_name = nil)
-			@target = table_name if table_name
-			@target
-		end
-		
-		def force_downcase
-			@force_case = :downcase
-		end
-		
-		def force_upcase
-			@force_case = :upcase
-		end
-
-		def target_model
-			DataGateway.model_for(@target)
-		end
-
-		# DSL syntax sugar for declare column mappers
-		def columns(column_map)
-			column_map.each_pair do |key, value|
-				case value
-					when Symbol, String: column_map[key] = write_to(value.to_sym)
-				end
-			end
-			@column_map.merge! column_map
-			
-			@column_map.each_pair do |key, value|
-				defaults = column_mapper_defaults(key)
-				@column_map[key].options.replace(defaults.merge!(@column_map[key].options))
-				#puts "map for #{key} => #{@column_map[key].options.inspect}"
-			end
-			@column_map
-		end
-
-		def none
-			create_column_mapper({:method => :none})
-		end
-
-		def write_to(column)
-			create_column_mapper({:method => :write_to, :write_to => column})
-		end
-
-		def lookup(options)
-			options[:method] = :lookup
-			create_column_mapper(options)
-		end
 
-		def identity_by(*cols)
-			@identity = cols
-		end
-		
-	 # utilities
-		def column_mapper_for(column_name)
-			@column_map[column_name] ||= create_column_mapper(column_mapper_defaults(column_name))
-		end
-		
-		def column_mapper_defaults(column_name)
-			column_name = column_name.to_s.send(@force_case).to_sym if @force_case
-			case column_name.to_s
-				when /(.*)_id$/i:	{	:method => :lookup,
-														:write_to => column_name,
-														:table => $1.downcase.camelize.to_sym,
-														:key => @job.get_mapper($1.downcase.camelize.to_sym).identity,
-														:value => :id
-													}
-			else 	{	:method => :write_to,
-							:write_to => column_name
-						}
-			end
-			
-		end		
-
-		def update_all(records)
-			DataGateway.logger.debug "importing #{records.size} recods into table #{source}"
-			klass = target_model
-			DataGateway.logger.debug "target class #{klass} (PK: #{klass.primary_key})"
-			records.each { |rec|
-				rec = map(rec)
-				update_rec(rec) unless current_fixups(rec)
-			}
-		end
-
-		def clear_fixups
-			@fixups = {}
-		end
-
-		def register_fixup(value, col_mapper)
-			@ar_fixups << [value, col_mapper]
-		end
-
-		def current_fixups(rec)
-			DataGateway.logger.debug "current fixups for: #{rec.inspect}"
-			DataGateway.logger.debug @ar_fixups.inspect
-			return false if @ar_fixups.empty?
-			@fixups[rec] ||= []
-			@fixups[rec].concat @ar_fixups
-			@ar_fixups = []
-			true
-		end
-
-		def solve_fixups
-			DataGateway.logger.debug "start solve_fixups #{@fixups.size}"
-			@fixups.each_pair { |record, data|
-				data.each{|item|
-					value = item[0]
-					col_mapper = item[1]
-					col_mapper.solve_fixup(record, value)
-				}
-				update_rec(record)
-			}
-			DataGateway.logger.debug "end solve_fixups"
-		ensure
-			clear_fixups
-		end
-
-		def update_rec(rec)
-			DataGateway.logger.debug "update rec class #{target_model} '->' #{rec.inspect}"
-			conditions = identity.map{|key| key.to_s}
-			values = conditions.map{|key| rec.delete(key.to_sym)}
-			finder = 'find_or_create_by_' + conditions.join('_')
-			DataGateway.logger.debug "finder: #{finder}(#{values.join(',')})"
-		
-			ar = target_model.send(finder, *values)
-			DataGateway.logger.debug "get: #{ar.inspect}"
-			DataGateway.logger.debug "upd atrt: #{rec.inspect}"
-			ar.update_attributes(rec)
-			return ar[:id]
-		end
-
-		def map(ar)
-			new_ar = {}
-			ar.each_pair do |key, value|
-				column_mapper_for(key).apply_to(new_ar, value)
-			end
-			new_ar
-		end
-	private
-		def create_column_mapper( options)
-			ColumnMapper.new(self, options)
-		end
-	end
-
-	class Job
-		attr_accessor :format_id
-		attr :transfer
-		attr :name
-		
-		def initialize(name, &block)
-			@name = name
-			@use_resources = DataGateway.use_resources
-			@encoder = NullEncoder.new
-			@transfer = nil
-			instance_eval(&block) if block_given?
-		end
-		
-		def transfer_for(options)		
-			options.merge! :logger => DataGateway.logger
-			@transfer = SSHTransfer.new(options)
-		end
-		
-		def run
-			AppConfig.logger.info "running job #{name}"
-			do_run
-			AppConfig.logger.info "job #{name} finished"
-		end
-
-	private	
-		# only for debug purpouse
-		def use_resources(bool)
-			@use_resources = bool
-		end
-		
-		def encode(options)
-			@encoder = Encoder.new(options)
-		end
-		
-		def pack(data)
-			DataGateway.conduit_for(self.format_id).pack(data)
-		end
-
-		def unpack(data)
-			DataGateway.conduit_for(format_id).unpack(data)
-		end
-
-		def dbchange_point
-			@dbchange_point ||= DBChangePoint.class_for(ActiveRecord::Base.connection).new
-		end
-	end
-	
-	class Import < Job
-		def initialize(name, &block)
-			@mappers = {} #import mappers
-			super
-		end
-
-		def do_run
-			transfer.download if transfer and transfer.download_from
-
-			Dir.glob(File.join(DataGateway::inbox, '*.zip')).sort.each do |filename|
-			DataGateway.logger.info "Importing #{filename}"
-			import_file(filename)
-			FileUtils.mv(filename, DataGateway::donebox)
-			DataGateway.logger.info "Import done for #{filename}"
-			end		
-		end
-
-		def get_mapper(table)
-			@mappers[table] ||= TableMapper.new(self, table)
-		end
-		
-	public # DSL
-
-		def download_from(options)
-			options.merge! 'local_files' => DataGateway.inbox
-			transfer_for(options)
-		end
-
-		def importing(table, &block)
-			get_mapper(table).instance_eval(&block) if block_given?
-		end
-
-	private
-
-		def import_file(file_name)
-			self.format_id = file_name
-			clear_fixups
-			begin
-				Zip::ZipInputStream::open(file_name) { |io|
-					while (entry = io.get_next_entry)
-						content = io.read
-						fname = entry.name
-						if fname =~ /^attach\//
-							fname = fname.gsub(/^attach\//, '')
-							import_attach(fname, content)
-						else
-							class_name = File.basename(fname, '.*').downcase.camelize.to_sym
-							import_class(class_name, content)
-						end
-					end
-				}
-			ensure
-				solve_fixups
-			end
-		end
-
-		def clear_fixups
-			@mappers.each_value {|mapper|
-				mapper.clear_fixups
-			}
-		end
-
-		def solve_fixups
-			@mappers.each_value {|mapper|
-				mapper.solve_fixups
-			}
-		end
-
-		def import_class(class_name, data)
-			DataGateway.logger.info "Importing data for #{class_name}"
-			mapper = get_mapper(class_name)
-			if mapper.ignore?
-				DataGateway.logger.info "Ignored"
-				return
-			end
-			mapper.update_all unpack(data)
-			#p dbchange_point.capture_for(klass)
-			DataGateway.logger.info "done for #{class_name}"
-		end
-
-		def import_attach(fname, content)
-			return unless @use_resources
-			DataGateway.logger.info "importing attach #{fname}"
-			target_file = File.expand_path(fname, DataGateway.resource_path)
-			FileUtils.mkdir_p(File.dirname(target_file))
-			File.open(target_file, "wb") {|f| f.write content}
-		end
-	end
-	
-	class Export < Job
-		def initialize(name, &block)
-			@exports = {}
-			@exports_order = []
-			export_to :csv #default format
-			super		
-		end
-		
-		def each
-			tables = @exports_order
-			tables.each { |cname|
-			  yield cname, export_class(cname)
-			}
-		end
-		
-		def export_folder
-			@export_folder ||= create_folder
-		end
-		
-		def close_export_folder
-			@export_folder.close if @export_folder
-		end
-		
-		def do_run
-			begin				
-				self.each { |cname, records|
-					file_name = "#{cname}.#{self.format_id}"
-					DataGateway.logger.debug "Exporting archive #{file_name}"
-					#records.each { |rec|
-					#		rec.each {|key, value|
-					#				rec[key] = @encoder.pack(rec[key]) unless (key =~ /^img/i)
-					#		}
-					#}
-					write_file(file_name, records)
-				}
-			ensure
-				close_export_folder
-			end
-			
-			transfer.upload if transfer and transfer.upload_to				
-		end
-	
-	private # DSL
-		def upload_to(options)
-			options.merge! 'local_files' => File.join(DataGateway.outbox, "*.zip")
-			transfer_for(options)
-		end
-		
-		def exporting(table_name, options = {})
-			@exports_order << table_name
-			@exports[table_name] = options
-		end
-
-		def export_to(format)
-			self.format_id = format
-		end
-  private
-		def create_folder(outbox = DataGateway::outbox)
-			FileUtils.mkpath(outbox)
-			stamp = Time.now.strftime("%Y%m%d%H%M%S")
-			folder_name = File.join(outbox, "#{stamp}.#{self.format_id}")
-			#(APP_ENV == 'development') ? FolderWriter.new(folder_name) :
-										ZipFolderWriter.new(folder_name)
-		end
-
-		def export_class(cname)
-			result = []
-			bm = Benchmark.measure {
-				klass = DataGateway.model_for(cname.to_s.downcase.camelize)
-				DataGateway.logger.info "Exporting #{klass} (#{self.format_id})"
-				records = klass.find(:all, @exports[cname])
-				records.each { |r| export_attachments(r) } if @use_resources
-				result = pack(records) unless records.empty?
-			}
-			DataGateway.logger.info "data exported in #{bm}"
-			result
-		end
-
-		def export_attachments(record)			
-			record.attributes.keys.select { |key| key =~ /^img/i }.each { |key|
-				file = record.attributes[key]
-				#TODO Si el path del attach es absoluto, quiza debemos tener cuidado al descomprimir el zip
-				next if file.blank?
-				path = File.expand_path(file, DataGateway.resource_path)
-				DataGateway.logger.info "exporting attach #{path}"
-				#TODO Si el fichero ya se exporto solo en este vuelta, no repetir!
-				next unless File.exist?(path)
-				attach_data = File.open(path, 'rb') { |f| f.read }
-				write_file(File.join('attach', file), attach_data, 'b')
-			}
-		end
-
-		def write_file(file_name, data, options = '')
-			op = StringIO.new("", "w")
-			op.puts data
-			if (options == 'b')
-				export_folder.open(file_name, "w" + options){|f| f.puts op.string}				
-			else
-				export_folder.open(file_name, "w" + options){|f| f.puts @encoder.pack(op.string)}
-			end
-		end
-	end
  public
-	# DataGateway DSL	
+
+	# DataGateway DSL
 	def resource_path(path= nil)
 		if path
 			@@resource_path = File.expand_path(path)
@@ -545,9 +60,13 @@ module DataGateway
 			@@resource_path || ''
 		end
 	end
-	
+
 	module_function(:resource_path)
-	
+
+  def last_time(key)
+  	DataGateway.last_time[key.to_s].utc.to_s(:db)
+  end
+
 	def db_connection(conn)
 		::ActiveRecord::Base.establish_connection conn
 		if conn['nax_empresa']
@@ -557,13 +76,11 @@ module DataGateway
 
 	def nax_connection(connection)
 		NAX::ActiveRecord::Base.establish_connection connection
-	end		
+	end
 
 	def import(name, &block)
 		name = name.to_sym
-		self.class.class_eval {
-			attr_reader name
-		}
+		self.class.class_eval {attr_reader name }
 		new_job = Import.new(name, &block)
 		instance_variable_set('@' + name.to_s, new_job)
 		return  new_job
@@ -571,25 +88,21 @@ module DataGateway
 
 	def export(name, &block)
 		name = name.to_sym
-		self.class.class_eval {
-			attr_reader name
-		}
+		self.class.class_eval { attr_reader name }
 		new_job = Export.new(name, &block)
 		instance_variable_set('@' + name.to_s, new_job)
 		return  new_job
 	end
-	
-	def upload_to(options)
-		Export.new('upload') {
-			upload_to options
-		}.run
 
+	def upload_to(options)
+		options.merge! :logger => DataGateway.logger
+		options.merge! 'local_files' => File.join(DataGateway.outbox, "*.zip")
+		SSHTransfer.new(options).upload
 	end
 
 	def download_from(options)
-		Import.new('download') {
-			download_from options
-		}.run
-
+		options.merge! :logger => DataGateway.logger
+		options.merge! 'local_files' => DataGateway.inbox
+		SSHTransfer.new(options).download
 	end
 end

data/lib/datagateway/encoder.rb

@@ -0,0 +1,27 @@
+module DataGateway
+
+	class NullEncoder
+		def pack(data)
+			data
+		end
+
+		def unpack(data)
+			data
+		end
+	end
+
+	class Encoder
+		def initialize(options)
+			@pack = Iconv.new(options[:to], options[:from])
+			@unpack = Iconv.new(options[:from], options[:to])
+		end
+
+		def pack(data)
+			@pack.iconv(data)
+		end
+
+		def unpack(data)
+			@unpack.iconv(data)
+		end
+	end
+end