data_imp 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 86e11f4b6817ad75532ca9af858c261295516645
4
- data.tar.gz: 7cdd8b503afa39ce898eaf8a77a5034f86af448e
3
+ metadata.gz: 57cc9c4696b447711d832623c57ffe9ae43be4b1
4
+ data.tar.gz: 0172aec2f0bba07abcb7898342922423fcbdfd1f
5
5
  SHA512:
6
- metadata.gz: 84f36d210ff7c43709501f10eed5289e2f7d4d965c2cb1fcfb0edca5134e2af08e5a0821b307ef22ee20c414e43e73398e26879b773ab2cbef33d9f7af7dba91
7
- data.tar.gz: 9735f74a69ccee71adb71b3b4bb28273525cbad80eb7158f337f7357a38bd83d413cc920b95ceea2fb2adf9157ee3d06f3af7a6e6aad98229b89cccc40fbbac0
6
+ metadata.gz: 4660ceb5a760d601720b5a7f69c9ec3db2b9f15ed7aa559ed1b9d6ca2b9a6d41a454a1b9dc52b03c50f867e413f8c7a91e8f082d1b74e5bc9a14809da1929034
7
+ data.tar.gz: 5647c5104d74dc14e9e0933c2f86b5198b3753f1adc0cc028dd9edb22341084d363f443d12bfe6868473b7656932e443598bd9591ef14db22b494becb7d25e1e
data/data_imp.gemspec CHANGED
@@ -22,9 +22,10 @@ Gem::Specification.new do |spec|
22
22
 
23
23
  spec.add_runtime_dependency 'activesupport'
24
24
 
25
- spec.add_development_dependency "bundler", "~> 1.14.6"
25
+ spec.add_development_dependency "bundler", "~> 1.15.4"
26
26
  spec.add_development_dependency "rake", "~> 10.0"
27
27
  spec.add_development_dependency "rspec", "~> 3.0"
28
28
  spec.add_development_dependency "roo" #, "~> 3.0"
29
29
  spec.add_development_dependency "roo-xls" #, "~> 3.0"
30
+ spec.add_development_dependency "tiny_tds"
30
31
  end
@@ -1,63 +1,8 @@
1
- require 'pathname'
2
1
 
3
2
  require_relative "parser"
4
3
  require_relative "porter"
4
+ require_relative "importer"
5
5
 
6
6
  module DataImp::ClassMethods
7
- def self.extended(mod)
8
- mod.delegate :data_dir, :find_parser, :find_importer, to: :class
9
- end
10
-
11
- def root= dir
12
- @root = dir
13
- end
14
-
15
- def root
16
- @root ||= Pathname.pwd
17
- end
18
-
19
- def data_dir= dir
20
- path = Pathname.new(dir)
21
- @data_dir =
22
- if path.absolute?
23
- path
24
- else
25
- root.join(path)
26
- end
27
- end
28
-
29
- def data_dir
30
- @data_dir ||= root.join('data')
31
- end
32
-
33
- def import(file=nil,*args,&block)
34
- file.strip!
35
- return if file =~ /^#/
36
- new(file, *args,&block).import
37
- end
38
-
39
- def import_list *args
40
- args.each do |arg|
41
- arg.each_line do |file|
42
- import file
43
- end
44
- end
45
- end
46
-
47
- def find_parser type
48
- return type if type.kind_of? DataImp::Parser
49
- type = type.to_s.camelize
50
- const_get "#{type}Parser"
51
- rescue NameError => e
52
- DataImp::Parser.find_parser type
53
- end
54
-
55
- def find_importer type
56
- return type if type.kind_of? DataImp::Porter
57
- type = type.to_s.camelize
58
- const_get "#{type}Importer"
59
- rescue NameError => e
60
- DataImp::Porter.find_importer type
61
- end
62
7
  end
63
8
 
@@ -0,0 +1,28 @@
1
+ require 'pathname'
2
+ module DataImp::Dir
3
+ def self.extended(mod)
4
+ mod.delegate :data_dir, to: :class
5
+ end
6
+
7
+ def root= dir
8
+ @root = dir
9
+ end
10
+
11
+ def root
12
+ @root ||= Pathname.pwd
13
+ end
14
+
15
+ def data_dir= dir
16
+ path = Pathname.new(dir)
17
+ @data_dir =
18
+ if path.absolute?
19
+ path
20
+ else
21
+ root.join(path)
22
+ end
23
+ end
24
+
25
+ def data_dir
26
+ @data_dir ||= root.join('data')
27
+ end
28
+ end
@@ -0,0 +1,29 @@
1
+ require_relative "parser"
2
+ require_relative "porter"
3
+ require_relative "importer"
4
+
5
+ module DataImp::Finders
6
+ def self.extended(mod)
7
+ mod.delegate :find_parser, :find_importer, to: :class
8
+ end
9
+
10
+ def self.included(mod)
11
+ mod.delegate :const_get, to: :class
12
+ end
13
+
14
+ def find_parser type
15
+ return type if type.kind_of? DataImp::Parser
16
+ type = type.to_s.camelize
17
+ const_get "#{type}Parser"
18
+ rescue NameError => e
19
+ DataImp::Parser.find_parser type
20
+ end
21
+
22
+ def find_importer type
23
+ return type if type.kind_of? DataImp::Porter
24
+ type = type.to_s.camelize
25
+ const_get "#{type}Importer"
26
+ rescue NameError => e
27
+ DataImp::Porter.find_importer type
28
+ end
29
+ end
@@ -0,0 +1,19 @@
1
+
2
+ require_relative "parser"
3
+ require_relative "porter"
4
+ require_relative "importer"
5
+
6
+ module DataImp::ImportMethods
7
+ def import(file=nil,*args,&block)
8
+ file.strip!
9
+ return if file =~ /^#/
10
+ new(file, *args,&block).import
11
+ end
12
+
13
+ def import_list list, *args, &block
14
+ list.each_line do |file|
15
+ import file, *args, &block
16
+ end
17
+ end
18
+ end
19
+
@@ -0,0 +1,5 @@
1
+
2
+ require_relative 'porter'
3
+ DataImp::Importer = DataImp::Porter
4
+ # class alias because someone is going to mix these up
5
+
@@ -0,0 +1,20 @@
1
+ class DataImp::Tds::Table
2
+ attr_reader :client, :table, :schema
3
+ def initialize client, table, schema: 'dbo'
4
+ @client = client
5
+ @table = table
6
+ @schema = schema
7
+ end
8
+
9
+ def sql
10
+ "select * from [#{schema}].[#{table}]"
11
+ end
12
+
13
+ def result
14
+ @result ||= client.execute(sql)
15
+ end
16
+
17
+ def each &block
18
+ result.each &block
19
+ end
20
+ end
@@ -0,0 +1,45 @@
1
+ require 'tiny_tds'
2
+ require_relative "finders"
3
+ class DataImp::Tds
4
+ extend DataImp::Finders
5
+
6
+ def initialize options={}
7
+ @options = options
8
+ end
9
+
10
+ def options
11
+ @options ||= {}
12
+ end
13
+
14
+ def client
15
+ @client ||= TinyTds::Client.new options
16
+ end
17
+
18
+ def import table
19
+ parts = table.split('.')
20
+ table = parts.pop
21
+ schema = parts.shift
22
+ importer = find_importer(table)
23
+ tbl = Table.new(client, table, schema: schema)
24
+ puts "Importing #{table} with #{importer}"
25
+ importer.before_all_imports
26
+ tbl.each do |hash, index|
27
+ porter = importer.new(hash,index)
28
+ begin
29
+ porter.before_import
30
+ porter.import
31
+ porter.after_import
32
+ show_progress index
33
+ rescue StandardError => e
34
+ warn "#{table}:#{index}:#{e.class.name}"
35
+ porter.on_error e
36
+ end
37
+ end
38
+ importer.after_all_imports
39
+ puts
40
+ end
41
+
42
+ def show_progress index
43
+ end
44
+ end
45
+ require_relative "tds/table"
@@ -1,3 +1,3 @@
1
1
  class DataImp
2
- VERSION = "0.1.0"
2
+ VERSION = "0.2.0"
3
3
  end
data/lib/data_imp.rb CHANGED
@@ -1,10 +1,16 @@
1
1
 
2
2
  require 'active_support/core_ext/string'
3
- require_relative "data_imp/version"
3
+
4
+ require_relative "data_imp/finders"
5
+ require_relative "data_imp/dir"
6
+ require_relative "data_imp/import_methods"
4
7
  require_relative "data_imp/class_methods"
5
8
 
6
9
  # Dir[File.dirname(__FILE__) + '/data_imp/*.rb'].each {|file| require file }
7
10
  class DataImp
11
+ extend DataImp::Finders
12
+ extend DataImp::Dir
13
+ extend DataImp::ImportMethods
8
14
  extend DataImp::ClassMethods
9
15
  attr_accessor :file, :parser, :importer, :extname, :basename
10
16
 
@@ -41,30 +47,5 @@ class DataImp
41
47
  print '.' if index % 100 == 0
42
48
  end
43
49
 
44
- # def import_file file
45
- #
46
- # parser_class_name = ext.classify
47
- # importer_class_name = base.concat('_importer').classify
48
- # file = data_dir.join(file)
49
- #
50
- # begin
51
- # parser = const_get(parser_class_name)
52
- # importer = importer_class_name.constantize
53
- # rescue NameError => e
54
- # warn "No importer class: #{e}"
55
- # return
56
- # end
57
- # parser.new(file, importer).import
58
- # end
59
-
60
- # def reader= reader
61
- # return unless reader
62
- # @reader =
63
- # case reader
64
- # when IO,DataImp::Reader then reader
65
- # when String then open(reader)
66
- # when Pathname then reader.open
67
- # end
68
- # end
69
50
  end
70
51
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: data_imp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Robert W. Ferney
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-09-13 00:00:00.000000000 Z
11
+ date: 2017-10-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -30,14 +30,14 @@ dependencies:
30
30
  requirements:
31
31
  - - "~>"
32
32
  - !ruby/object:Gem::Version
33
- version: 1.14.6
33
+ version: 1.15.4
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
38
  - - "~>"
39
39
  - !ruby/object:Gem::Version
40
- version: 1.14.6
40
+ version: 1.15.4
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: rake
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
94
94
  - - ">="
95
95
  - !ruby/object:Gem::Version
96
96
  version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: tiny_tds
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
97
111
  description:
98
112
  email:
99
113
  - rob@ferney.org
@@ -122,6 +136,10 @@ files:
122
136
  - data_imp.gemspec
123
137
  - lib/data_imp.rb
124
138
  - lib/data_imp/class_methods.rb
139
+ - lib/data_imp/dir.rb
140
+ - lib/data_imp/finders.rb
141
+ - lib/data_imp/import_methods.rb
142
+ - lib/data_imp/importer.rb
125
143
  - lib/data_imp/no_importer.rb
126
144
  - lib/data_imp/no_parser.rb
127
145
  - lib/data_imp/parser.rb
@@ -135,6 +153,8 @@ files:
135
153
  - lib/data_imp/parser/yaml.rb
136
154
  - lib/data_imp/porter.rb
137
155
  - lib/data_imp/porter/sample.rb
156
+ - lib/data_imp/tds.rb
157
+ - lib/data_imp/tds/table.rb
138
158
  - lib/data_imp/version.rb
139
159
  homepage: https://github.com/capnregex/data_imp
140
160
  licenses: