data_imp 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/data_imp.gemspec +2 -1
- data/lib/data_imp/class_methods.rb +1 -56
- data/lib/data_imp/dir.rb +28 -0
- data/lib/data_imp/finders.rb +29 -0
- data/lib/data_imp/import_methods.rb +19 -0
- data/lib/data_imp/importer.rb +5 -0
- data/lib/data_imp/tds/table.rb +20 -0
- data/lib/data_imp/tds.rb +45 -0
- data/lib/data_imp/version.rb +1 -1
- data/lib/data_imp.rb +7 -26
- metadata +24 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 57cc9c4696b447711d832623c57ffe9ae43be4b1
|
4
|
+
data.tar.gz: 0172aec2f0bba07abcb7898342922423fcbdfd1f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4660ceb5a760d601720b5a7f69c9ec3db2b9f15ed7aa559ed1b9d6ca2b9a6d41a454a1b9dc52b03c50f867e413f8c7a91e8f082d1b74e5bc9a14809da1929034
|
7
|
+
data.tar.gz: 5647c5104d74dc14e9e0933c2f86b5198b3753f1adc0cc028dd9edb22341084d363f443d12bfe6868473b7656932e443598bd9591ef14db22b494becb7d25e1e
|
data/data_imp.gemspec
CHANGED
@@ -22,9 +22,10 @@ Gem::Specification.new do |spec|
|
|
22
22
|
|
23
23
|
spec.add_runtime_dependency 'activesupport'
|
24
24
|
|
25
|
-
spec.add_development_dependency "bundler", "~> 1.
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.15.4"
|
26
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
27
27
|
spec.add_development_dependency "rspec", "~> 3.0"
|
28
28
|
spec.add_development_dependency "roo" #, "~> 3.0"
|
29
29
|
spec.add_development_dependency "roo-xls" #, "~> 3.0"
|
30
|
+
spec.add_development_dependency "tiny_tds"
|
30
31
|
end
|
@@ -1,63 +1,8 @@
|
|
1
|
-
require 'pathname'
|
2
1
|
|
3
2
|
require_relative "parser"
|
4
3
|
require_relative "porter"
|
4
|
+
require_relative "importer"
|
5
5
|
|
6
6
|
module DataImp::ClassMethods
|
7
|
-
def self.extended(mod)
|
8
|
-
mod.delegate :data_dir, :find_parser, :find_importer, to: :class
|
9
|
-
end
|
10
|
-
|
11
|
-
def root= dir
|
12
|
-
@root = dir
|
13
|
-
end
|
14
|
-
|
15
|
-
def root
|
16
|
-
@root ||= Pathname.pwd
|
17
|
-
end
|
18
|
-
|
19
|
-
def data_dir= dir
|
20
|
-
path = Pathname.new(dir)
|
21
|
-
@data_dir =
|
22
|
-
if path.absolute?
|
23
|
-
path
|
24
|
-
else
|
25
|
-
root.join(path)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def data_dir
|
30
|
-
@data_dir ||= root.join('data')
|
31
|
-
end
|
32
|
-
|
33
|
-
def import(file=nil,*args,&block)
|
34
|
-
file.strip!
|
35
|
-
return if file =~ /^#/
|
36
|
-
new(file, *args,&block).import
|
37
|
-
end
|
38
|
-
|
39
|
-
def import_list *args
|
40
|
-
args.each do |arg|
|
41
|
-
arg.each_line do |file|
|
42
|
-
import file
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def find_parser type
|
48
|
-
return type if type.kind_of? DataImp::Parser
|
49
|
-
type = type.to_s.camelize
|
50
|
-
const_get "#{type}Parser"
|
51
|
-
rescue NameError => e
|
52
|
-
DataImp::Parser.find_parser type
|
53
|
-
end
|
54
|
-
|
55
|
-
def find_importer type
|
56
|
-
return type if type.kind_of? DataImp::Porter
|
57
|
-
type = type.to_s.camelize
|
58
|
-
const_get "#{type}Importer"
|
59
|
-
rescue NameError => e
|
60
|
-
DataImp::Porter.find_importer type
|
61
|
-
end
|
62
7
|
end
|
63
8
|
|
data/lib/data_imp/dir.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
module DataImp::Dir
|
3
|
+
def self.extended(mod)
|
4
|
+
mod.delegate :data_dir, to: :class
|
5
|
+
end
|
6
|
+
|
7
|
+
def root= dir
|
8
|
+
@root = dir
|
9
|
+
end
|
10
|
+
|
11
|
+
def root
|
12
|
+
@root ||= Pathname.pwd
|
13
|
+
end
|
14
|
+
|
15
|
+
def data_dir= dir
|
16
|
+
path = Pathname.new(dir)
|
17
|
+
@data_dir =
|
18
|
+
if path.absolute?
|
19
|
+
path
|
20
|
+
else
|
21
|
+
root.join(path)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def data_dir
|
26
|
+
@data_dir ||= root.join('data')
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative "parser"
|
2
|
+
require_relative "porter"
|
3
|
+
require_relative "importer"
|
4
|
+
|
5
|
+
module DataImp::Finders
|
6
|
+
def self.extended(mod)
|
7
|
+
mod.delegate :find_parser, :find_importer, to: :class
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.included(mod)
|
11
|
+
mod.delegate :const_get, to: :class
|
12
|
+
end
|
13
|
+
|
14
|
+
def find_parser type
|
15
|
+
return type if type.kind_of? DataImp::Parser
|
16
|
+
type = type.to_s.camelize
|
17
|
+
const_get "#{type}Parser"
|
18
|
+
rescue NameError => e
|
19
|
+
DataImp::Parser.find_parser type
|
20
|
+
end
|
21
|
+
|
22
|
+
def find_importer type
|
23
|
+
return type if type.kind_of? DataImp::Porter
|
24
|
+
type = type.to_s.camelize
|
25
|
+
const_get "#{type}Importer"
|
26
|
+
rescue NameError => e
|
27
|
+
DataImp::Porter.find_importer type
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
require_relative "parser"
|
3
|
+
require_relative "porter"
|
4
|
+
require_relative "importer"
|
5
|
+
|
6
|
+
module DataImp::ImportMethods
|
7
|
+
def import(file=nil,*args,&block)
|
8
|
+
file.strip!
|
9
|
+
return if file =~ /^#/
|
10
|
+
new(file, *args,&block).import
|
11
|
+
end
|
12
|
+
|
13
|
+
def import_list list, *args, &block
|
14
|
+
list.each_line do |file|
|
15
|
+
import file, *args, &block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
class DataImp::Tds::Table
|
2
|
+
attr_reader :client, :table, :schema
|
3
|
+
def initialize client, table, schema: 'dbo'
|
4
|
+
@client = client
|
5
|
+
@table = table
|
6
|
+
@schema = schema
|
7
|
+
end
|
8
|
+
|
9
|
+
def sql
|
10
|
+
"select * from [#{schema}].[#{table}]"
|
11
|
+
end
|
12
|
+
|
13
|
+
def result
|
14
|
+
@result ||= client.execute(sql)
|
15
|
+
end
|
16
|
+
|
17
|
+
def each &block
|
18
|
+
result.each &block
|
19
|
+
end
|
20
|
+
end
|
data/lib/data_imp/tds.rb
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'tiny_tds'
|
2
|
+
require_relative "finders"
|
3
|
+
class DataImp::Tds
|
4
|
+
extend DataImp::Finders
|
5
|
+
|
6
|
+
def initialize options={}
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def options
|
11
|
+
@options ||= {}
|
12
|
+
end
|
13
|
+
|
14
|
+
def client
|
15
|
+
@client ||= TinyTds::Client.new options
|
16
|
+
end
|
17
|
+
|
18
|
+
def import table
|
19
|
+
parts = table.split('.')
|
20
|
+
table = parts.pop
|
21
|
+
schema = parts.shift
|
22
|
+
importer = find_importer(table)
|
23
|
+
tbl = Table.new(client, table, schema: schema)
|
24
|
+
puts "Importing #{table} with #{importer}"
|
25
|
+
importer.before_all_imports
|
26
|
+
tbl.each do |hash, index|
|
27
|
+
porter = importer.new(hash,index)
|
28
|
+
begin
|
29
|
+
porter.before_import
|
30
|
+
porter.import
|
31
|
+
porter.after_import
|
32
|
+
show_progress index
|
33
|
+
rescue StandardError => e
|
34
|
+
warn "#{table}:#{index}:#{e.class.name}"
|
35
|
+
porter.on_error e
|
36
|
+
end
|
37
|
+
end
|
38
|
+
importer.after_all_imports
|
39
|
+
puts
|
40
|
+
end
|
41
|
+
|
42
|
+
def show_progress index
|
43
|
+
end
|
44
|
+
end
|
45
|
+
require_relative "tds/table"
|
data/lib/data_imp/version.rb
CHANGED
data/lib/data_imp.rb
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
|
2
2
|
require 'active_support/core_ext/string'
|
3
|
-
|
3
|
+
|
4
|
+
require_relative "data_imp/finders"
|
5
|
+
require_relative "data_imp/dir"
|
6
|
+
require_relative "data_imp/import_methods"
|
4
7
|
require_relative "data_imp/class_methods"
|
5
8
|
|
6
9
|
# Dir[File.dirname(__FILE__) + '/data_imp/*.rb'].each {|file| require file }
|
7
10
|
class DataImp
|
11
|
+
extend DataImp::Finders
|
12
|
+
extend DataImp::Dir
|
13
|
+
extend DataImp::ImportMethods
|
8
14
|
extend DataImp::ClassMethods
|
9
15
|
attr_accessor :file, :parser, :importer, :extname, :basename
|
10
16
|
|
@@ -41,30 +47,5 @@ class DataImp
|
|
41
47
|
print '.' if index % 100 == 0
|
42
48
|
end
|
43
49
|
|
44
|
-
# def import_file file
|
45
|
-
#
|
46
|
-
# parser_class_name = ext.classify
|
47
|
-
# importer_class_name = base.concat('_importer').classify
|
48
|
-
# file = data_dir.join(file)
|
49
|
-
#
|
50
|
-
# begin
|
51
|
-
# parser = const_get(parser_class_name)
|
52
|
-
# importer = importer_class_name.constantize
|
53
|
-
# rescue NameError => e
|
54
|
-
# warn "No importer class: #{e}"
|
55
|
-
# return
|
56
|
-
# end
|
57
|
-
# parser.new(file, importer).import
|
58
|
-
# end
|
59
|
-
|
60
|
-
# def reader= reader
|
61
|
-
# return unless reader
|
62
|
-
# @reader =
|
63
|
-
# case reader
|
64
|
-
# when IO,DataImp::Reader then reader
|
65
|
-
# when String then open(reader)
|
66
|
-
# when Pathname then reader.open
|
67
|
-
# end
|
68
|
-
# end
|
69
50
|
end
|
70
51
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_imp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert W. Ferney
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 1.
|
33
|
+
version: 1.15.4
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 1.
|
40
|
+
version: 1.15.4
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: tiny_tds
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description:
|
98
112
|
email:
|
99
113
|
- rob@ferney.org
|
@@ -122,6 +136,10 @@ files:
|
|
122
136
|
- data_imp.gemspec
|
123
137
|
- lib/data_imp.rb
|
124
138
|
- lib/data_imp/class_methods.rb
|
139
|
+
- lib/data_imp/dir.rb
|
140
|
+
- lib/data_imp/finders.rb
|
141
|
+
- lib/data_imp/import_methods.rb
|
142
|
+
- lib/data_imp/importer.rb
|
125
143
|
- lib/data_imp/no_importer.rb
|
126
144
|
- lib/data_imp/no_parser.rb
|
127
145
|
- lib/data_imp/parser.rb
|
@@ -135,6 +153,8 @@ files:
|
|
135
153
|
- lib/data_imp/parser/yaml.rb
|
136
154
|
- lib/data_imp/porter.rb
|
137
155
|
- lib/data_imp/porter/sample.rb
|
156
|
+
- lib/data_imp/tds.rb
|
157
|
+
- lib/data_imp/tds/table.rb
|
138
158
|
- lib/data_imp/version.rb
|
139
159
|
homepage: https://github.com/capnregex/data_imp
|
140
160
|
licenses:
|