jcangas-datagateway 1.2.2 → 1.4.4

data/lib/datagateway/job.rb

@@ -0,0 +1,223 @@
+require 'encoder'
+require 'mapper'
+
+module DataGateway
+	class Job
+		attr_accessor :format_id
+		attr :name
+
+		def initialize(name, &block)
+			@name = name
+			@use_resources = DataGateway.use_resources
+			@encoder = NullEncoder.new
+			instance_eval(&block) if block_given?
+		end
+
+		def run
+			DataGateway.logger.info "running job #{name}"
+			do_run
+			DataGateway.logger.info "job #{name} finished"
+		end
+
+	private
+		def use_resources(bool)
+			@use_resources = bool
+		end
+
+		def encode(options)
+			@encoder = Encoder.new(options)
+		end
+
+		def pack(data)
+			DataGateway.conduit_for(self.format_id).pack(data)
+		end
+
+		def unpack(data)
+			DataGateway.conduit_for(format_id).unpack(data)
+		end
+
+		def dbchange_point
+			@dbchange_point ||= DBChangePoint.class_for(ActiveRecord::Base.connection).new
+		end
+	end
+
+	class Import < Job
+		def initialize(name, &block)
+			@mappers = {} #import mappers
+			super
+		end
+
+		def do_run
+			new_last_time = Time.now
+			Dir.glob(File.join(DataGateway::inbox, '*.zip')).sort.each do |filename|
+				DataGateway.logger.info "Importing #{filename}"
+				import_file(filename)
+				FileUtils.mv(filename, DataGateway::donebox)
+				DataGateway.logger.info "Import done for #{filename}"
+			end
+			DataGateway.last_time['import'] = new_last_time
+		end
+		
+		def donebox
+			Dir.glob(File.join(DataGateway::donebox, '*.zip')).each do |filename|
+				FileUtils.rm(filename) if File.mtime(filename) < 7.days.ago
+			end		
+		end
+		
+		def get_mapper(table)
+			@mappers[table] ||= TableMapper.new(self, table)
+		end
+
+	public # DSL
+
+		def importing(table, &block)
+			get_mapper(table).instance_eval(&block) if block_given?
+		end
+
+		def import_file(file_name)
+			self.format_id = file_name
+			clear_fixups
+			begin
+				Zip::ZipInputStream::open(file_name) { |io|
+					while (entry = io.get_next_entry)
+						content = io.read
+						fname = entry.name
+						if fname =~ /^attach\//
+							fname = fname.gsub(/^attach\//, '')
+							import_attach(fname, content)
+						else
+							class_name = File.basename(fname, '.*').downcase.camelize.to_sym
+							import_class(class_name, content)
+						end
+					end
+				}
+			ensure
+				solve_fixups
+			end
+		end
+
+		def clear_fixups
+			@mappers.each_value {|mapper| mapper.clear_fixups	}
+		end
+
+		def solve_fixups
+			@mappers.each_value {|mapper|	mapper.solve_fixups	}
+		end
+
+		def import_class(class_name, data)
+			DataGateway.logger.info "Importing data for #{class_name}"
+			mapper = get_mapper(class_name)
+			if mapper.ignore?
+				DataGateway.logger.info "Ignored"
+				return
+			end
+			mapper.update_all unpack(data)
+			#p dbchange_point.capture_for(klass)
+			DataGateway.logger.info "done for #{class_name}"
+		end
+
+		def import_attach(fname, content)
+			return unless @use_resources
+			DataGateway.logger.info "importing attach #{fname}"
+			target_file = File.expand_path(fname, DataGateway.resource_path)
+			FileUtils.mkdir_p(File.dirname(target_file))
+			File.open(target_file, "wb") {|f| f.write content}
+		end
+	end
+
+	class Export < Job
+		def initialize(name, &block)
+			@exports = {}
+			@exports_order = []
+			export_to :csv #default format
+			super
+		end
+
+		def each
+			tables = @exports_order
+			tables.each { |cname|
+			  yield cname, export_class(cname)
+			}
+		end
+
+		def export_folder
+			@export_folder ||= create_folder
+		end
+
+		def close_export_folder
+			@export_folder.close if @export_folder
+		end
+
+		def do_run
+			new_last_time = Time.now
+			begin
+				self.each { |cname, records|
+					file_name = "#{cname}.#{self.format_id}"
+					DataGateway.logger.debug "Exporting archive #{file_name}"
+					write_file(file_name, records)
+				}
+			ensure
+				close_export_folder
+			end
+			DataGateway.last_time['export'] = new_last_time
+		end
+
+	private # DSL
+
+		def exporting(table_name, options = {})
+			@exports_order << table_name
+			@exports[table_name] = options
+		end
+
+		def export_to(format)
+			self.format_id = format
+		end
+
+  private
+		def create_folder(outbox = DataGateway::outbox)
+			FileUtils.mkpath(outbox)
+			stamp = Time.now.strftime("%Y%m%d%H%M%S")
+			folder_name = File.join(outbox, "#{stamp}.#{self.format_id}")
+			#(APP_ENV == 'development') ? FolderWriter.new(folder_name) :
+										ZipFolderWriter.new(folder_name)
+		end
+
+		def export_class(cname)
+			result = []
+			bm = Benchmark.measure {
+				klass = DataGateway.model_for(cname.to_s.downcase.camelize)
+				DataGateway.logger.info "Exporting #{klass} (#{self.format_id})"
+				records = klass.find(:all, @exports[cname])
+				records.each { |r| export_attachments(r) } if @use_resources
+				result = pack(records) unless records.empty?
+			}
+			DataGateway.logger.info "data exported in #{bm}"
+			result
+		end
+
+		def export_attachments(record)
+			record.attributes.keys.select { |key| key =~ /^img/i }.each { |key|
+				file = record.attributes[key]
+				next if file.blank?
+				path = File.expand_path(file, DataGateway.resource_path)
+				#TODO: Si el fichero ya se exporto solo en este vuelta, no repetir!
+				next unless File.exist?(path)
+				modified = File.mtime(path) > DataGateway.last_time['export']
+				next unless modified
+				DataGateway.logger.info "exporting attach #{path}"
+				attach_data = File.open(path, 'rb') { |f| f.read }
+				write_file(File.join('attach', file), attach_data, 'b')
+			}
+		end
+
+		def write_file(file_name, data, options = '')
+			op = StringIO.new("", "w")
+			op.puts data
+			if (options == 'b')
+				export_folder.open(file_name, "w" + options){|f| f.puts op.string}
+			else
+				export_folder.open(file_name, "w" + options){|f| f.puts @encoder.pack(op.string)}
+			end
+		end
+	end
+end

data/lib/datagateway/jobs/nexus_export.rb

@@ -5,19 +5,20 @@ db_connection AppConfig.connections[:nexusdb]
 export :nexus_export do
 	encode( :to => "UTF-8" , :from => "WINDOWS-1252")
 	export_to :csv
-	exporting :user, :conditions => 'CODCLI IS NOT NULL'
-	exporting :category 
+	exporting :user,
+		:select => 'USERID AS LOGIN, CODCLI AS CODE, EMAIL',
+		:conditions => 'CODCLI IS NOT NULL'
+	exporting :category
 	exporting :family
 
 	exporting :news,
-		:select => 'DESCRIPTION AS TITLE, FROMDATE AS PUBLISHED_AT, TODATE AS EXPIRES_AT, PRIORITY, NEWSTEXT BODY'
-	exporting :product, # :limit => 5,
+		:select => 'ID AS CODE, DESCRIPTION AS TITLE, FROMDATE AS PUBLISHED_AT, TODATE AS EXPIRES_AT, PRIORITY, NEWSTEXT BODY'
+	exporting :product,
 		:select => 'CATEGORY_ID, FAMILY_ID, CODE, IMG_DETAIL, IMG_MINI, DETAIL, da.DESCART AS LABEL, PRCVENTA',
 		:joins => 'as p left outer join ARTICULO as ar on CODE = ar.CODART left outer join DESCRIPA as da on CODE = da.CODART',
 		:conditions => "da.codidioma = 'CAS'"
-		
+
 	exporting :tarifa_venta,
 		:select => 'IDTARIFAV, CODART, FECMAX, PRECIO, TARIFA, UNIDADES',
 		:joins => 'as tarf inner join mmproducts p on code = tarf.codart'
 end
-

data/lib/datagateway/jobs/nexus_import.rb

@@ -7,10 +7,7 @@ import(:nexus_import) do
 	importing :User do
 		force_upcase
 		identity_by :USERID
-		columns :full_name => none
-				#:userid => write_to(:USERID),
-				#:email => :EMAIL,
-				
+		columns :full_name => none				
 	end
 	
 	##FIX: no puede ser LineItem pq hay varios documentos -> LineOrder

data/lib/datagateway/jobs/shopin_export.rb

@@ -3,14 +3,17 @@ db_connection AppConfig.connections[:shopindb]
 
 export 'shopin_export' do
 	exporting :user,
-		:select => 'login as userid, email',
+		:select => "login as userid, email, ' 1' as codcli",
 		:conditions => 'activated_at is not null'
 
 	exporting :order,
 		:select => 'number, reference, special_instructions, login as userid',
-		:joins => 'as ord left outer join users as usr on ord.user_id = usr.id'
-		
+		:joins => 'as ord left outer join users as usr on ord.user_id = usr.id',
+		:conditions => "(ord.updated_at > '#{last_time(:export)}')"
+
 	exporting :line_items,
-    	:select => 'line.id as id, number as order_id, price, product_id, quantity',
-		:joins => 'as line left outer join orders as ord on line.order_id = ord.id'
-end
+    :select => 'line.id as id, number as order_id, price, product_id, quantity',
+		:joins => 'as line left outer join orders as ord on line.order_id = ord.id',
+		:conditions => "(ord.updated_at > '#{last_time(:export)}')"
+
+end

data/lib/datagateway/jobs/shopin_import.rb

@@ -18,16 +18,17 @@ import 'shopin_import' do
 		columns :DESCRIPTION => :name,
 						:DBREV => none
 	end
-	
+
 	importing :News do
+		identity_by :code
+    force_downcase
     target :Post
-    force_downcase      
 	end
-	
+
 	importing :User do
-		ignore!
+		identity_by :login
 	end
-	
+
 	importing :Product do
 		identity_by :code
 		force_downcase
@@ -35,7 +36,7 @@ import 'shopin_import' do
 				:DETAIL => :description,
 				:PRCVENTA => :price
 	end
-	
+
 	importing :TarifaVenta do
 		target :PriceList
 		identity_by :code
@@ -45,6 +46,6 @@ import 'shopin_import' do
 			:FECMAX => :expires_at,
 			:PRECIO => :price,
 			:TARIFA => :group,
-			:UNIDADES => :quantity		
+			:UNIDADES => :quantity
 	end
-end
+end

data/lib/datagateway/mapper.rb

@@ -0,0 +1,226 @@
+module DataGateway
+	class ColumnMapper
+		attr :options
+		def initialize(table_mapper, options = {})
+			@table_mapper = table_mapper
+			@options = options
+		end
+
+		def apply_to(record, value)
+			DataGateway.logger.debug "apply_to(#{record.inspect}, #{value}) options #{@options.inspect}"
+			send(@options[:method], record, value)
+		end
+
+		# value mappers
+		def none(record, value)
+			# do nothing
+		end
+
+		def write_to(record, value)
+			record[@options[:write_to]] = value
+		end
+
+		def lookup(record, value)
+
+			result_status = {}
+			new_value = lookup_map(value, result_status)
+			if result_status[:found]
+				write_to(record, new_value)
+			else
+				@table_mapper.register_fixup(value, self)
+			end
+		end
+
+	private
+		def	lookup_map(value, result_status)
+			DataGateway.logger.debug "lookup_map(#{value}, #{result_status}) options #{@options.inspect}"
+			lookup_model = DataGateway.model_for(@options[:table])
+			finder = "find_or_create_by_#{@options[:key]}"
+			lookup_rec = lookup_model.send(finder, value)
+			result_status[:found] = !(lookup_rec.nil?)
+			if result_status[:found]
+				lookup_rec.send(@options[:value]).to_s # valor mapeado
+			else
+				nil
+			end
+		end
+
+		def solve_fixup(record, value)
+			DataGateway.logger.debug "mapper #{@map_method} opt. #{@options.inspect}"
+			result_status = {}
+			DataGateway.logger.debug "lookup #{record.inspect} -> <#{value}>"
+			value = lookup_map(value, result_status)
+			if result_status[:found]
+				DataGateway.logger.debug "FOUND: #{value}"
+				write_to(record, value)
+			else
+				DataGateway.logger.debug "NOT FOUND!"
+			end
+		end
+	end
+
+	class TableMapper
+		attr :source
+		attr :identity
+
+		def initialize(job, table)
+			@job = job
+			@source = table
+			@target = table
+			@column_map = {}
+			@identity = [:id]
+			@ar_fixups = []
+			@force_case = false
+			clear_fixups
+		end
+
+		def ignore!
+			@target = nil
+		end
+
+		def ignore?
+			@target == nil
+		end
+
+		def target(table_name = nil)
+			@target = table_name if table_name
+			@target
+		end
+
+		def force_downcase
+			@force_case = :downcase
+		end
+
+		def force_upcase
+			@force_case = :upcase
+		end
+
+		def target_model
+			DataGateway.model_for(@target)
+		end
+
+		# DSL syntax sugar for declare column mappers
+		def columns(column_map)
+			column_map.each_pair do |key, value|
+				case value
+					when Symbol, String: column_map[key] = write_to(value.to_sym)
+				end
+			end
+			@column_map.merge! column_map
+
+			@column_map.each_pair do |key, value|
+				defaults = column_mapper_defaults(key)
+				@column_map[key].options.replace(defaults.merge!(@column_map[key].options))
+				#puts "map for #{key} => #{@column_map[key].options.inspect}"
+			end
+			@column_map
+		end
+
+		def none
+			create_column_mapper({:method => :none})
+		end
+
+		def write_to(column)
+			create_column_mapper({:method => :write_to, :write_to => column})
+		end
+
+		def lookup(options)
+			options[:method] = :lookup
+			create_column_mapper(options)
+		end
+
+		def identity_by(*cols)
+			@identity = cols
+		end
+
+	 # utilities
+		def column_mapper_for(column_name)
+			@column_map[column_name] ||= create_column_mapper(column_mapper_defaults(column_name))
+		end
+
+		def column_mapper_defaults(column_name)
+			column_name = column_name.to_s.send(@force_case).to_sym if @force_case
+			case column_name.to_s
+				when /(.*)_id$/i:	{	:method => :lookup,
+														:write_to => column_name,
+														:table => $1.downcase.camelize.to_sym,
+														:key => @job.get_mapper($1.downcase.camelize.to_sym).identity,
+														:value => :id
+													}
+			else 	{	:method => :write_to,
+							:write_to => column_name
+						}
+			end
+
+		end
+
+		def update_all(records)
+			DataGateway.logger.debug "importing #{records.size} recods into table #{source}"
+			klass = target_model
+			DataGateway.logger.debug "target class #{klass} (PK: #{klass.primary_key})"
+			records.each { |rec|
+				rec = map(rec)
+				update_rec(rec) unless current_fixups(rec)
+			}
+		end
+
+		def clear_fixups
+			@fixups = {}
+		end
+
+		def register_fixup(value, col_mapper)
+			@ar_fixups << [value, col_mapper]
+		end
+
+		def current_fixups(rec)
+			DataGateway.logger.debug "current fixups for: #{rec.inspect}"
+			DataGateway.logger.debug @ar_fixups.inspect
+			return false if @ar_fixups.empty?
+			@fixups[rec] ||= []
+			@fixups[rec].concat @ar_fixups
+			@ar_fixups = []
+			true
+		end
+
+		def solve_fixups
+			DataGateway.logger.debug "start solve_fixups #{@fixups.size}"
+			@fixups.each_pair { |record, data|
+				data.each{|item|
+					value = item[0]
+					col_mapper = item[1]
+					col_mapper.solve_fixup(record, value)
+				}
+				update_rec(record)
+			}
+			DataGateway.logger.debug "end solve_fixups"
+		ensure
+			clear_fixups
+		end
+
+		def update_rec(rec)
+			DataGateway.logger.debug "update rec class #{target_model} '->' #{rec.inspect}"
+			conditions = identity.map{|key| key.to_s}
+			values = conditions.map{|key| rec.delete(key.to_sym)}
+			finder = 'find_or_create_by_' + conditions.join('_')
+			DataGateway.logger.debug "finder: #{finder}(#{values.join(',')})"
+
+			ar = target_model.send(finder, *values)
+			DataGateway.logger.debug "get: #{ar.inspect}"
+			DataGateway.logger.debug "upd atrt: #{rec.inspect}"
+			ar.update_attributes(rec)
+			return ar[:id]
+		end
+
+		def map(ar)
+			new_ar = {}
+			ar.each_pair do |key, value|
+				column_mapper_for(key).apply_to(new_ar, value)
+			end
+			new_ar
+		end
+	private
+		def create_column_mapper( options)
+			ColumnMapper.new(self, options)
+		end
+	end
+end