jcangas-datagateway 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION.yml +4 -0
- data/bin/datagw +35 -0
- data/lib/datagateway/conduits/csv_conduit.rb +20 -0
- data/lib/datagateway/conduits/yaml_conduit.rb +9 -0
- data/lib/datagateway/data_gateway.rb +582 -0
- data/lib/datagateway/dbchange_point.rb +27 -0
- data/lib/datagateway/folder_writer.rb +50 -0
- data/lib/datagateway/jobs/nexus_export.rb +31 -0
- data/lib/datagateway/jobs/nexus_import.rb +61 -0
- data/lib/datagateway/jobs/shopin_export.rb +19 -0
- data/lib/datagateway/jobs/shopin_import.rb +52 -0
- data/lib/datagateway/nexus/nax_ar.rb +174 -0
- data/lib/datagateway/nexus/nax_tlb.rb +13157 -0
- data/lib/datagateway/nexus/nexus_ar.rb +145 -0
- data/lib/datagateway/ssh_transfer.rb +80 -0
- data/lib/datagateway/version.rb +13 -0
- data/lib/datagateway.rb +294 -0
- data/lib/version.rb +16 -0
- data/template-prj/README +0 -0
- data/template-prj/config/README +4 -0
- data/template-prj/config/database.yml +32 -0
- data/template-prj/config/settings.yml +25 -0
- data/template-prj/data/donebox/README +1 -0
- data/template-prj/data/inbox/README +1 -0
- data/template-prj/data/outbox/README +1 -0
- data/template-prj/datagateway.rb +18 -0
- data/template-prj/log/README +2 -0
- data/test/datagateway_test.rb +7 -0
- data/test/test_helper.rb +10 -0
- metadata +100 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
require 'fileutils'
|
|
3
|
+
|
|
4
|
+
class ZipFolderWriter
|
|
5
|
+
|
|
6
|
+
def initialize(root_name)
|
|
7
|
+
@root_name = root_name + '.zip'
|
|
8
|
+
@export_zos = Zip::ZipOutputStream.open(@root_name)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def folder_name
|
|
12
|
+
@root_name
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def open(file_name, options)
|
|
16
|
+
@export_zos.put_next_entry(file_name)
|
|
17
|
+
begin
|
|
18
|
+
yield @export_zos
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def close
|
|
23
|
+
@export_zos.close
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
class FolderWriter
|
|
28
|
+
def initialize(root_name)
|
|
29
|
+
FileUtils.mkpath(root_name)
|
|
30
|
+
@root_name = root_name
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def folder_name
|
|
34
|
+
@root_name
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def open(file_name, options)
|
|
38
|
+
file_name = File.join(@root_name, file_name)
|
|
39
|
+
FileUtils.mkdir_p(File.dirname(file_name))
|
|
40
|
+
f = File.open(file_name, options)
|
|
41
|
+
begin
|
|
42
|
+
yield f
|
|
43
|
+
ensure
|
|
44
|
+
f.close
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def close
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require 'nexus/nexus_ar'
|
|
2
|
+
|
|
3
|
+
db_connection AppConfig.connections[:nexusdb]
|
|
4
|
+
|
|
5
|
+
export(:transfer) do
|
|
6
|
+
upload_to AppConfig.sftp
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
export(:nexus_export) do
|
|
10
|
+
encode( :to => "UTF-8" , :from => "WINDOWS-1252")
|
|
11
|
+
export_to :csv
|
|
12
|
+
exporting :user, :conditions => 'CODCLI IS NOT NULL'
|
|
13
|
+
exporting :category #, :limit => 5
|
|
14
|
+
exporting :family #, :limit => 5
|
|
15
|
+
exporting :tarifa_venta,
|
|
16
|
+
:select => 'IDTARIFAV, CODART, FECMAX, PRECIO, TARIFA, UNIDADES',
|
|
17
|
+
:joins => 'as tarf inner join mmproducts p on code = tarf.codart'
|
|
18
|
+
|
|
19
|
+
exporting :news,
|
|
20
|
+
:select => 'DESCRIPTION AS TITLE, FROMDATE AS PUBLISHED_AT, TODATE AS EXPIRES_AT, PRIORITY, NEWSTEXT BODY'
|
|
21
|
+
exporting :product, # :limit => 5,
|
|
22
|
+
:select => 'CATEGORY_ID, FAMILY_ID, CODE, IMG_DETAIL, IMG_MINI, DETAIL, da.DESCART AS LABEL, PRCVENTA',
|
|
23
|
+
:joins => 'as p left outer join ARTICULO as ar on CODE = ar.CODART left outer join DESCRIPA as da on CODE = da.CODART',
|
|
24
|
+
:conditions => "da.codidioma = 'CAS'"
|
|
25
|
+
|
|
26
|
+
upload_to AppConfig.sftp
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
nexus_export.run
|
|
30
|
+
#transfer.run
|
|
31
|
+
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
|
|
2
|
+
require 'nexus/nexus_ar'
|
|
3
|
+
|
|
4
|
+
db_connection AppConfig.connections[:nexusdb]
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
import(:nexus_import) do
|
|
8
|
+
download_from AppConfig.sftp
|
|
9
|
+
|
|
10
|
+
importing :User do
|
|
11
|
+
force_upcase
|
|
12
|
+
identity_by :USERID
|
|
13
|
+
columns :full_name => none
|
|
14
|
+
#:userid => write_to(:USERID),
|
|
15
|
+
#:email => :EMAIL,
|
|
16
|
+
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
##FIX: no puede ser LineItem pq hay varios documentos -> LineOrder
|
|
20
|
+
importing :LineItems do
|
|
21
|
+
force_upcase
|
|
22
|
+
identity_by :PARAM1
|
|
23
|
+
columns :id => :PARAM1,
|
|
24
|
+
:product_id => :CODART,
|
|
25
|
+
:price => :PRCMONEDA,
|
|
26
|
+
:quantity => :UNIDADES,
|
|
27
|
+
:order_id => lookup( :write_to => :IDPEDV,
|
|
28
|
+
#:table => :Order,
|
|
29
|
+
:key => :PARAM1,
|
|
30
|
+
:value => :IDPEDV)
|
|
31
|
+
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
importing :Order do
|
|
35
|
+
force_upcase
|
|
36
|
+
identity_by :PARAM1
|
|
37
|
+
columns :userid => lookup( :write_to => :CODCLI,
|
|
38
|
+
:table => :User, :key => :USERID,
|
|
39
|
+
:value => :CODCLI),
|
|
40
|
+
:tax_amount => none,
|
|
41
|
+
:item_total => none,
|
|
42
|
+
:ip_address => none,
|
|
43
|
+
:created_at => none,
|
|
44
|
+
:item_total => none,
|
|
45
|
+
:reference => :REFERENCIA,
|
|
46
|
+
:status => none,
|
|
47
|
+
:ship_amount => none,
|
|
48
|
+
:total => none,
|
|
49
|
+
:updated_at => none,
|
|
50
|
+
:ship_method => none,
|
|
51
|
+
:id => none,
|
|
52
|
+
:number => :PARAM1,
|
|
53
|
+
:special_instructions => :OBSERVACIONES
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
begin
|
|
58
|
+
nexus_import.run
|
|
59
|
+
ensure
|
|
60
|
+
NAX::ActiveRecord::Base.close
|
|
61
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
|
|
2
|
+
db_connection AppConfig.connections[:shopindb]
|
|
3
|
+
resource_path AppConfig.path('../public/images/db/')
|
|
4
|
+
|
|
5
|
+
export 'shopin_export' do
|
|
6
|
+
exporting :user,
|
|
7
|
+
:select => 'login as userid, email',
|
|
8
|
+
:conditions => 'activated_at is not null'
|
|
9
|
+
|
|
10
|
+
exporting :order,
|
|
11
|
+
:select => 'number, reference, special_instructions, login as userid',
|
|
12
|
+
:joins => 'as ord left outer join users as usr on ord.user_id = usr.id'
|
|
13
|
+
|
|
14
|
+
exporting :line_items,
|
|
15
|
+
:select => 'line.id as id, number as order_id, price, product_id, quantity',
|
|
16
|
+
:joins => 'as line left outer join orders as ord on line.order_id = ord.id'
|
|
17
|
+
|
|
18
|
+
run
|
|
19
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
#require 'nexus/nax_ar'
|
|
2
|
+
|
|
3
|
+
db_connection AppConfig.connections[:shopindb]
|
|
4
|
+
resource_path AppConfig.path('../public/images/db/')
|
|
5
|
+
|
|
6
|
+
import 'shopin_import' do
|
|
7
|
+
|
|
8
|
+
importing :Family do
|
|
9
|
+
identity_by :code
|
|
10
|
+
force_downcase
|
|
11
|
+
columns :DESCRIPTION => :name,
|
|
12
|
+
:DBREV => none
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
importing :Category do
|
|
16
|
+
identity_by :code
|
|
17
|
+
force_downcase
|
|
18
|
+
columns :DESCRIPTION => :name,
|
|
19
|
+
:DBREV => none
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
importing :News do
|
|
23
|
+
target :Post
|
|
24
|
+
force_downcase
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
importing :User do
|
|
28
|
+
ignore!
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
importing :Product do
|
|
32
|
+
identity_by :code
|
|
33
|
+
force_downcase
|
|
34
|
+
columns :DBREV => none,
|
|
35
|
+
:DETAIL => :description,
|
|
36
|
+
:PRCVENTA => :price
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
importing :TarifaVenta do
|
|
40
|
+
target :PriceList
|
|
41
|
+
identity_by :code
|
|
42
|
+
force_downcase
|
|
43
|
+
columns :IDTARIFAV => :code,
|
|
44
|
+
:CODART => lookup( :write_to => :product_id, :table => :Product, :key => :code, :value => :id),
|
|
45
|
+
:FECMAX => :expires_at,
|
|
46
|
+
:PRECIO => :price,
|
|
47
|
+
:TARIFA => :group,
|
|
48
|
+
:UNIDADES => :quantity
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
run
|
|
52
|
+
end
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
|
|
2
|
+
base = File.dirname(__FILE__)
|
|
3
|
+
require File.expand_path('nax_tlb', base)
|
|
4
|
+
|
|
5
|
+
module NAX #:nodoc: all
|
|
6
|
+
class DummyLogger
|
|
7
|
+
def method_missing(name, *args)
|
|
8
|
+
return unless [:debug, :info, :warn, :error, :fatal].include?(name)
|
|
9
|
+
puts args.join(' ')
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
DYNA_METHODS = /^(find_by|find_or_create_by)(\w+)/
|
|
14
|
+
module ActiveRecord
|
|
15
|
+
class Base
|
|
16
|
+
@@enlace = nil
|
|
17
|
+
@@connection = nil
|
|
18
|
+
class << self
|
|
19
|
+
attr_accessor :logger
|
|
20
|
+
def logger
|
|
21
|
+
@logger ||= DummyLogger.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def establish_connection(connection)
|
|
25
|
+
@@connection = connection
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def current_connection
|
|
29
|
+
@@connection
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def connected?
|
|
33
|
+
@@enlace
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def connect
|
|
37
|
+
return if connected?
|
|
38
|
+
logger.debug "Conectando NAX a #{@@connection.inspect}"
|
|
39
|
+
@@enlace = NAX_Enlace.new
|
|
40
|
+
@@enlace.RaiseOnException = false
|
|
41
|
+
@@enlace.LoginUsuario(@@connection['username'], @@connection['password'])
|
|
42
|
+
@@enlace.Iniciar(@@connection['nax_empresa'])
|
|
43
|
+
logger.debug 'NAX connectado ' + @@enlace.inspect
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def check
|
|
47
|
+
puts "check #{@@enlace.bError} Nro #{@@enlace.nError} M #{@@enlace.sMensaje} "
|
|
48
|
+
raise @@enlace.sMensaje if @@enlace.bError
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def close
|
|
52
|
+
if connected?
|
|
53
|
+
logger.debug "Cerrando NAX de #{@@connection.inspect}"
|
|
54
|
+
enlace = @@enlace
|
|
55
|
+
@@enlace = nil
|
|
56
|
+
enlace.Acabar
|
|
57
|
+
end
|
|
58
|
+
logger.debug "NAX cerrado"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
module InstanceMethods
|
|
64
|
+
attr_writer :update_status
|
|
65
|
+
attr_writer :attributes
|
|
66
|
+
|
|
67
|
+
def attributes
|
|
68
|
+
@attributes ||= {}
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def update_status
|
|
72
|
+
@update_status ||= []
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def logger
|
|
76
|
+
NAX::ActiveRecord::Base.logger
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def update_attributes(values)
|
|
80
|
+
logger.debug "(NAX) update_attributes #{self}<#{values.inspect}>"
|
|
81
|
+
attributes.merge! values
|
|
82
|
+
save
|
|
83
|
+
logger.debug "NAX update completed"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def save
|
|
87
|
+
logger.debug "(NAX) #{self}.save: #{attributes.inspect})"
|
|
88
|
+
if update_status.include?(:us_created)
|
|
89
|
+
internal_create
|
|
90
|
+
update_status.delete :us_created
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
module ClassMethods
|
|
98
|
+
def table_name=(value)
|
|
99
|
+
@table_name = value
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def table_name
|
|
103
|
+
@table_name
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def primary_key=(value)
|
|
107
|
+
@primary_key = value
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def primary_key
|
|
111
|
+
@primary_key
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def logger
|
|
115
|
+
NAX::ActiveRecord::Base.logger
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def ensure_connected
|
|
119
|
+
Base.connect
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def check
|
|
123
|
+
Base.check
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def find(id)
|
|
127
|
+
self.find_by(:id => id)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def respond_to?(method)
|
|
131
|
+
return true if DYNA_METHODS.match(method.to_s)
|
|
132
|
+
super
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def method_missing(method, *args, &block)
|
|
136
|
+
md = DYNA_METHODS.match(method.to_s)
|
|
137
|
+
|
|
138
|
+
if md
|
|
139
|
+
base_method = md[1].to_sym
|
|
140
|
+
field_names = md[2].split('and').map{|s| s.gsub!('_','').to_sym}
|
|
141
|
+
criteria = Hash[*field_names.zip(args).flatten]
|
|
142
|
+
send(base_method, criteria)
|
|
143
|
+
else
|
|
144
|
+
super
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def create(values={})
|
|
149
|
+
AppConfig.logger.debug "(NAX) #{self}.create #{values.inspect}"
|
|
150
|
+
result = new
|
|
151
|
+
result.update_status << :us_created
|
|
152
|
+
result.attributes.merge! values
|
|
153
|
+
return result
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def find_or_create_by(criteria)
|
|
157
|
+
ensure_connected
|
|
158
|
+
AppConfig.logger.debug "(NAX) #{self}.find_or_create_by(#{criteria.inspect})"
|
|
159
|
+
find_by(criteria) || create(criteria)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def find_by(criteria)
|
|
163
|
+
AppConfig.logger.debug "(NAX) #{self}.find_by(#{criteria.inspect})"
|
|
164
|
+
nil
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
class Base
|
|
169
|
+
include InstanceMethods
|
|
170
|
+
extend ClassMethods
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|