data_imp 0.1.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/data_imp.gemspec +2 -1
- data/lib/data_imp/class_methods.rb +1 -56
- data/lib/data_imp/dir.rb +28 -0
- data/lib/data_imp/finders.rb +29 -0
- data/lib/data_imp/import_methods.rb +19 -0
- data/lib/data_imp/importer.rb +5 -0
- data/lib/data_imp/parser/csv.rb +1 -1
- data/lib/data_imp/tds/table.rb +24 -0
- data/lib/data_imp/tds.rb +53 -0
- data/lib/data_imp/version.rb +1 -1
- data/lib/data_imp.rb +10 -26
- metadata +25 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4c6777102891d246c75858d7cb7b60bc85c880990e6564a81bddc871241ae9d1
|
4
|
+
data.tar.gz: fc0e1a5a3d6aa1ec738fbd3f8dfb9b51a58185a08f9831e25f359f60c5724a54
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 21ef743537443b66646dbf1558f5cbbb22b712c131a1aecd7318282c31a3ecc2ebc8bc5db4d101651678387b757ae6ae699abd6f19fcd3aaaccf3ca646081bcc
|
7
|
+
data.tar.gz: efa9c8697c30157e4685a465371cae73c24c7d8ea60260bdf50c346559b6161ad4567ac6e64fa08979964ed49ce5163724186e31e3dfc746940d7ed8f6c28d23
|
data/data_imp.gemspec
CHANGED
@@ -22,9 +22,10 @@ Gem::Specification.new do |spec|
|
|
22
22
|
|
23
23
|
spec.add_runtime_dependency 'activesupport'
|
24
24
|
|
25
|
-
spec.add_development_dependency "bundler", "~>
|
25
|
+
spec.add_development_dependency "bundler", "~> 2.3.7"
|
26
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
27
27
|
spec.add_development_dependency "rspec", "~> 3.0"
|
28
28
|
spec.add_development_dependency "roo" #, "~> 3.0"
|
29
29
|
spec.add_development_dependency "roo-xls" #, "~> 3.0"
|
30
|
+
spec.add_development_dependency "tiny_tds"
|
30
31
|
end
|
@@ -1,63 +1,8 @@
|
|
1
|
-
require 'pathname'
|
2
1
|
|
3
2
|
require_relative "parser"
|
4
3
|
require_relative "porter"
|
4
|
+
require_relative "importer"
|
5
5
|
|
6
6
|
module DataImp::ClassMethods
|
7
|
-
def self.extended(mod)
|
8
|
-
mod.delegate :data_dir, :find_parser, :find_importer, to: :class
|
9
|
-
end
|
10
|
-
|
11
|
-
def root= dir
|
12
|
-
@root = dir
|
13
|
-
end
|
14
|
-
|
15
|
-
def root
|
16
|
-
@root ||= Pathname.pwd
|
17
|
-
end
|
18
|
-
|
19
|
-
def data_dir= dir
|
20
|
-
path = Pathname.new(dir)
|
21
|
-
@data_dir =
|
22
|
-
if path.absolute?
|
23
|
-
path
|
24
|
-
else
|
25
|
-
root.join(path)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def data_dir
|
30
|
-
@data_dir ||= root.join('data')
|
31
|
-
end
|
32
|
-
|
33
|
-
def import(file=nil,*args,&block)
|
34
|
-
file.strip!
|
35
|
-
return if file =~ /^#/
|
36
|
-
new(file, *args,&block).import
|
37
|
-
end
|
38
|
-
|
39
|
-
def import_list *args
|
40
|
-
args.each do |arg|
|
41
|
-
arg.each_line do |file|
|
42
|
-
import file
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def find_parser type
|
48
|
-
return type if type.kind_of? DataImp::Parser
|
49
|
-
type = type.to_s.camelize
|
50
|
-
const_get "#{type}Parser"
|
51
|
-
rescue NameError => e
|
52
|
-
DataImp::Parser.find_parser type
|
53
|
-
end
|
54
|
-
|
55
|
-
def find_importer type
|
56
|
-
return type if type.kind_of? DataImp::Porter
|
57
|
-
type = type.to_s.camelize
|
58
|
-
const_get "#{type}Importer"
|
59
|
-
rescue NameError => e
|
60
|
-
DataImp::Porter.find_importer type
|
61
|
-
end
|
62
7
|
end
|
63
8
|
|
data/lib/data_imp/dir.rb
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
module DataImp::Dir
|
3
|
+
def self.extended(mod)
|
4
|
+
mod.delegate :data_dir, to: :class
|
5
|
+
end
|
6
|
+
|
7
|
+
def root= dir
|
8
|
+
@root = dir
|
9
|
+
end
|
10
|
+
|
11
|
+
def root
|
12
|
+
@root ||= Pathname.pwd
|
13
|
+
end
|
14
|
+
|
15
|
+
def data_dir= dir
|
16
|
+
path = Pathname.new(dir)
|
17
|
+
@data_dir =
|
18
|
+
if path.absolute?
|
19
|
+
path
|
20
|
+
else
|
21
|
+
root.join(path)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def data_dir
|
26
|
+
@data_dir ||= root.join('data')
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative "parser"
|
2
|
+
require_relative "porter"
|
3
|
+
require_relative "importer"
|
4
|
+
|
5
|
+
module DataImp::Finders
|
6
|
+
def self.extended(mod)
|
7
|
+
mod.delegate :find_parser, :find_importer, to: :class
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.included(mod)
|
11
|
+
mod.delegate :const_get, to: :class
|
12
|
+
end
|
13
|
+
|
14
|
+
def find_parser type
|
15
|
+
return type if type.kind_of? DataImp::Parser
|
16
|
+
type = type.to_s.camelize
|
17
|
+
const_get "#{type}Parser"
|
18
|
+
rescue NameError => e
|
19
|
+
DataImp::Parser.find_parser type
|
20
|
+
end
|
21
|
+
|
22
|
+
def find_importer type
|
23
|
+
return type if type.kind_of? DataImp::Porter
|
24
|
+
type = type.to_s.camelize
|
25
|
+
const_get "#{type}Importer"
|
26
|
+
rescue NameError => e
|
27
|
+
DataImp::Porter.find_importer type
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
|
2
|
+
require_relative "parser"
|
3
|
+
require_relative "porter"
|
4
|
+
require_relative "importer"
|
5
|
+
|
6
|
+
module DataImp::ImportMethods
|
7
|
+
def import(file=nil, **args, &block)
|
8
|
+
file.strip!
|
9
|
+
return if file =~ /^#/
|
10
|
+
new(file, **args, &block).import
|
11
|
+
end
|
12
|
+
|
13
|
+
def import_list(list, **args, &block)
|
14
|
+
list.each_line do |file|
|
15
|
+
import file, **args, &block
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
data/lib/data_imp/parser/csv.rb
CHANGED
@@ -0,0 +1,24 @@
|
|
1
|
+
class DataImp::Tds::Table
|
2
|
+
attr_reader :client, :table, :schema
|
3
|
+
def initialize client, table, schema:
|
4
|
+
@client = client
|
5
|
+
@table = table
|
6
|
+
@schema = schema
|
7
|
+
end
|
8
|
+
|
9
|
+
def sql
|
10
|
+
if schema
|
11
|
+
"select * from [#{schema}].[#{table}]"
|
12
|
+
else
|
13
|
+
"select * from [#{table}]"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def result
|
18
|
+
@result ||= client.execute(sql)
|
19
|
+
end
|
20
|
+
|
21
|
+
def each &block
|
22
|
+
result.each &block
|
23
|
+
end
|
24
|
+
end
|
data/lib/data_imp/tds.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'tiny_tds'
|
2
|
+
require_relative "finders"
|
3
|
+
class DataImp::Tds
|
4
|
+
extend DataImp::Finders
|
5
|
+
|
6
|
+
def initialize options={}
|
7
|
+
@options = options
|
8
|
+
end
|
9
|
+
|
10
|
+
def options
|
11
|
+
@options ||= {}
|
12
|
+
end
|
13
|
+
|
14
|
+
def client
|
15
|
+
@client ||= TinyTds::Client.new options
|
16
|
+
end
|
17
|
+
|
18
|
+
def import table
|
19
|
+
table.strip!
|
20
|
+
return if table =~ /^#/
|
21
|
+
parts = table.split('.')
|
22
|
+
table = parts.pop
|
23
|
+
schema = parts.shift
|
24
|
+
importer = find_importer(table)
|
25
|
+
tbl = Table.new(client, table, schema: schema)
|
26
|
+
puts "Importing #{table} with #{importer}"
|
27
|
+
importer.before_all_imports
|
28
|
+
tbl.each do |hash, index|
|
29
|
+
porter = importer.new(hash,index)
|
30
|
+
begin
|
31
|
+
porter.before_import
|
32
|
+
porter.import
|
33
|
+
porter.after_import
|
34
|
+
show_progress index
|
35
|
+
rescue StandardError => e
|
36
|
+
warn "#{table}:#{index}:#{e.class.name}"
|
37
|
+
porter.on_error e
|
38
|
+
end
|
39
|
+
end
|
40
|
+
importer.after_all_imports
|
41
|
+
puts
|
42
|
+
end
|
43
|
+
|
44
|
+
def import_list list, *args, &block
|
45
|
+
list.each_line do |table|
|
46
|
+
import table, *args, &block
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def show_progress index
|
51
|
+
end
|
52
|
+
end
|
53
|
+
require_relative "tds/table"
|
data/lib/data_imp/version.rb
CHANGED
data/lib/data_imp.rb
CHANGED
@@ -1,10 +1,19 @@
|
|
1
1
|
|
2
2
|
require 'active_support/core_ext/string'
|
3
|
-
|
3
|
+
|
4
|
+
class DataImp
|
5
|
+
end
|
6
|
+
|
7
|
+
require_relative "data_imp/finders"
|
8
|
+
require_relative "data_imp/dir"
|
9
|
+
require_relative "data_imp/import_methods"
|
4
10
|
require_relative "data_imp/class_methods"
|
5
11
|
|
6
12
|
# Dir[File.dirname(__FILE__) + '/data_imp/*.rb'].each {|file| require file }
|
7
13
|
class DataImp
|
14
|
+
extend DataImp::Finders
|
15
|
+
extend DataImp::Dir
|
16
|
+
extend DataImp::ImportMethods
|
8
17
|
extend DataImp::ClassMethods
|
9
18
|
attr_accessor :file, :parser, :importer, :extname, :basename
|
10
19
|
|
@@ -41,30 +50,5 @@ class DataImp
|
|
41
50
|
print '.' if index % 100 == 0
|
42
51
|
end
|
43
52
|
|
44
|
-
# def import_file file
|
45
|
-
#
|
46
|
-
# parser_class_name = ext.classify
|
47
|
-
# importer_class_name = base.concat('_importer').classify
|
48
|
-
# file = data_dir.join(file)
|
49
|
-
#
|
50
|
-
# begin
|
51
|
-
# parser = const_get(parser_class_name)
|
52
|
-
# importer = importer_class_name.constantize
|
53
|
-
# rescue NameError => e
|
54
|
-
# warn "No importer class: #{e}"
|
55
|
-
# return
|
56
|
-
# end
|
57
|
-
# parser.new(file, importer).import
|
58
|
-
# end
|
59
|
-
|
60
|
-
# def reader= reader
|
61
|
-
# return unless reader
|
62
|
-
# @reader =
|
63
|
-
# case reader
|
64
|
-
# when IO,DataImp::Reader then reader
|
65
|
-
# when String then open(reader)
|
66
|
-
# when Pathname then reader.open
|
67
|
-
# end
|
68
|
-
# end
|
69
53
|
end
|
70
54
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: data_imp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert W. Ferney
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-08-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -30,14 +30,14 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 2.3.7
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 2.3.7
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +94,20 @@ dependencies:
|
|
94
94
|
- - ">="
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: tiny_tds
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
description:
|
98
112
|
email:
|
99
113
|
- rob@ferney.org
|
@@ -122,6 +136,10 @@ files:
|
|
122
136
|
- data_imp.gemspec
|
123
137
|
- lib/data_imp.rb
|
124
138
|
- lib/data_imp/class_methods.rb
|
139
|
+
- lib/data_imp/dir.rb
|
140
|
+
- lib/data_imp/finders.rb
|
141
|
+
- lib/data_imp/import_methods.rb
|
142
|
+
- lib/data_imp/importer.rb
|
125
143
|
- lib/data_imp/no_importer.rb
|
126
144
|
- lib/data_imp/no_parser.rb
|
127
145
|
- lib/data_imp/parser.rb
|
@@ -135,6 +153,8 @@ files:
|
|
135
153
|
- lib/data_imp/parser/yaml.rb
|
136
154
|
- lib/data_imp/porter.rb
|
137
155
|
- lib/data_imp/porter/sample.rb
|
156
|
+
- lib/data_imp/tds.rb
|
157
|
+
- lib/data_imp/tds/table.rb
|
138
158
|
- lib/data_imp/version.rb
|
139
159
|
homepage: https://github.com/capnregex/data_imp
|
140
160
|
licenses:
|
@@ -155,8 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
175
|
- !ruby/object:Gem::Version
|
156
176
|
version: '0'
|
157
177
|
requirements: []
|
158
|
-
|
159
|
-
rubygems_version: 2.5.1
|
178
|
+
rubygems_version: 3.3.7
|
160
179
|
signing_key:
|
161
180
|
specification_version: 4
|
162
181
|
summary: Scaffolding to make importing data easy
|