csv2psql 0.0.10 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gemspec +1 -0
- data/config/config.json +8 -0
- data/data/sample_bool.csv +13 -0
- data/lib/csv2psql/analyzer/analyzer.rb +86 -20
- data/lib/csv2psql/analyzer/types/base_analyzer.rb +36 -0
- data/lib/csv2psql/analyzer/types/bigint.rb +10 -30
- data/lib/csv2psql/analyzer/types/boolean.rb +31 -0
- data/lib/csv2psql/analyzer/types/character.rb +7 -17
- data/lib/csv2psql/analyzer/types/decimal.rb +15 -32
- data/lib/csv2psql/analyzer/types/null.rb +7 -17
- data/lib/csv2psql/analyzer/types/string.rb +7 -24
- data/lib/csv2psql/analyzer/types/uuid.rb +9 -18
- data/lib/csv2psql/cache/cache.rb +22 -0
- data/lib/csv2psql/cli/app.rb +4 -4
- data/lib/csv2psql/cli/cmd/analyze_cmd.rb +2 -2
- data/lib/csv2psql/cli/shared.rb +4 -4
- data/lib/csv2psql/config/config.rb +24 -0
- data/lib/csv2psql/convert/convert.rb +0 -5
- data/lib/csv2psql/extensions/string.rb +1 -1
- data/lib/csv2psql/frontend/base.rb +10 -0
- data/lib/csv2psql/frontend/csv.rb +19 -0
- data/lib/csv2psql/frontend/frontend.rb +9 -0
- data/lib/csv2psql/{dialects → generator/dialects}/psql.rb +0 -0
- data/lib/csv2psql/generator/generator.rb +0 -2
- data/lib/csv2psql/helpers/config_helper.rb +14 -0
- data/lib/csv2psql/helpers/json_helper.rb +31 -0
- data/lib/csv2psql/output/output.rb +0 -5
- data/lib/csv2psql/processor/processor.rb +31 -20
- data/lib/csv2psql/version.rb +2 -2
- data/spec/helpers/cli_helper.rb +2 -2
- metadata +34 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5299e63f4ab0f21575bc04d1b0f4e863f3121197
|
4
|
+
data.tar.gz: 32532a0c828576f90c016f51a880756c4161ee71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b227bf6c07fff2c0beafe9bfdba44654aaf372c2852ebf4d7430ac95cbd5478fc7cf2d4c0b26d0a168f551bd753f8997ebf17d54b8b59896b3614fe1df9cc394
|
7
|
+
data.tar.gz: 41595c98914f724d354b530e5fa9a21082d3d03b490e99e6cfb5b1efe6c60be1214b164f38403353d448bf42cc67bdf8a9fe93dde3ee4c5f955d9c3afb9b253b
|
data/.gemspec
CHANGED
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
|
|
25
25
|
|
26
26
|
s.add_dependency 'gli', '~> 2.11', '>= 2.11.0'
|
27
27
|
s.add_dependency 'json_pure', '~> 1.8.1'
|
28
|
+
s.add_dependency 'lru', '~> 0.1', '>= 0.1.0'
|
28
29
|
s.add_dependency 'multi_json', '~> 1.10.0'
|
29
30
|
s.add_dependency 'rake', '~> 10.3', '>= 10.3.2'
|
30
31
|
s.add_dependency 'terminal-table', '~> 1.4', '>= 1.4.5'
|
data/config/config.json
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
id,Firstname,Lastname,Address.Street,Address.City,Address.Details.Note,Married
|
2
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,True
|
3
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,False
|
4
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,1
|
5
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,0
|
6
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,blah
|
7
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,no
|
8
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"0"
|
9
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"1"
|
10
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"True"
|
11
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"False"
|
12
|
+
12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"blah"
|
13
|
+
45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"no"
|
@@ -1,7 +1,5 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
-
require 'csv'
|
4
|
-
require 'multi_json'
|
5
3
|
require 'pathname'
|
6
4
|
require 'pp'
|
7
5
|
|
@@ -10,14 +8,16 @@ require_relative '../extensions/string'
|
|
10
8
|
|
11
9
|
module Csv2Psql
|
12
10
|
# Analyzer file analyzer class
|
13
|
-
class Analyzer
|
11
|
+
class Analyzer # rubocop:disable Metrics/ClassLength
|
14
12
|
DEFAULT_OPTIONS = {}
|
15
13
|
ANALYZERS_DIR = File.join(File.dirname(__FILE__), 'types')
|
14
|
+
EXCLUDED_ANALYZERS = ['base_analyzer']
|
16
15
|
|
17
16
|
attr_reader :analyzers, :files
|
18
17
|
|
19
|
-
def initialize
|
18
|
+
def initialize(cache = nil)
|
20
19
|
@files = {}
|
20
|
+
@cache = cache
|
21
21
|
@analyzers = load_analyzers
|
22
22
|
end
|
23
23
|
|
@@ -25,23 +25,52 @@ module Csv2Psql
|
|
25
25
|
data = get_data(path)
|
26
26
|
|
27
27
|
header = CsvHelper.get_header(row, opts)
|
28
|
+
analyze_row(header, row, data)
|
29
|
+
|
30
|
+
data[:lines] = data[:lines] + 1
|
31
|
+
end
|
32
|
+
|
33
|
+
def analyze_column(analyzer, val, opts = { use_cache: false })
|
34
|
+
if opts[:use_cache]
|
35
|
+
res = cached_result(val) do
|
36
|
+
analyzer[:class].analyze(val)
|
37
|
+
end
|
38
|
+
else
|
39
|
+
res = analyzer[:class].analyze(val)
|
40
|
+
end
|
41
|
+
|
42
|
+
update_results(analyzer, res, val) if res
|
43
|
+
end
|
44
|
+
|
45
|
+
def analyze_row(header, row, data)
|
28
46
|
header.each do |h|
|
29
47
|
col = get_column(data, h)
|
30
|
-
val = row[h]
|
31
48
|
col.each do |_name, analyzer|
|
32
|
-
analyzer
|
49
|
+
analyze_column(analyzer, row[h])
|
33
50
|
end
|
34
51
|
end
|
52
|
+
end
|
35
53
|
|
36
|
-
|
54
|
+
def cached_result(val, &_block)
|
55
|
+
res = @cache.get(val)
|
56
|
+
if res.nil?
|
57
|
+
res = Proc.new.call(val)
|
58
|
+
@cache.put(val, res)
|
59
|
+
end
|
60
|
+
res
|
37
61
|
end
|
38
62
|
|
63
|
+
# Create column analyzers
|
39
64
|
def create_column(data, column)
|
40
65
|
data[:columns][column] = {}
|
41
66
|
res = data[:columns][column]
|
42
67
|
|
43
68
|
analyzers.each do |analyzer|
|
44
|
-
|
69
|
+
analyzer_class = analyzer[:class]
|
70
|
+
res[analyzer[:name]] = {
|
71
|
+
class: analyzer_class.new,
|
72
|
+
results: create_results(analyzer_class)
|
73
|
+
}
|
45
74
|
end
|
46
75
|
|
47
76
|
res
|
@@ -56,6 +85,17 @@ module Csv2Psql
|
|
56
85
|
files[path]
|
57
86
|
end
|
58
87
|
|
88
|
+
def create_results(analyzer_class)
|
89
|
+
res = {
|
90
|
+
count: 0
|
91
|
+
}
|
92
|
+
|
93
|
+
res[:min] = nil if analyzer_class.numeric?
|
94
|
+
res[:max] = nil if analyzer_class.numeric?
|
95
|
+
|
96
|
+
res
|
97
|
+
end
|
98
|
+
|
59
99
|
def get_data(path)
|
60
100
|
return files[path] if files.key?(path)
|
61
101
|
|
@@ -69,23 +109,49 @@ module Csv2Psql
|
|
69
109
|
create_column(data, column)
|
70
110
|
end
|
71
111
|
|
72
|
-
def
|
112
|
+
def load_analyzer_class(analyzer_class)
|
73
113
|
Object.const_get('Csv2Psql')
|
74
|
-
|
75
|
-
|
114
|
+
.const_get('Analyzers')
|
115
|
+
.const_get(analyzer_class)
|
116
|
+
end
|
117
|
+
|
118
|
+
def load_analyzer(path)
|
119
|
+
fname = File.basename(path, '.rb')
|
120
|
+
analyzer_class = fname.camel_case
|
121
|
+
require(path)
|
122
|
+
|
123
|
+
{
|
124
|
+
name: analyzer_class,
|
125
|
+
class: load_analyzer_class(analyzer_class)
|
126
|
+
}
|
76
127
|
end
|
77
128
|
|
78
129
|
def load_analyzers
|
79
|
-
Dir[ANALYZERS_DIR + '**/*.rb'].map do |path|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
{
|
85
|
-
name: analyzer_class,
|
86
|
-
class: load_analyze_class(analyzer_class)
|
87
|
-
}
|
130
|
+
res = Dir[ANALYZERS_DIR + '**/*.rb'].map do |path|
|
131
|
+
name = File.basename(path, '.rb')
|
132
|
+
next if EXCLUDED_ANALYZERS.include?(name)
|
133
|
+
load_analyzer(path)
|
88
134
|
end
|
135
|
+
|
136
|
+
res.compact
|
137
|
+
end
|
138
|
+
|
139
|
+
# Update numeric results
|
140
|
+
# @param ac analyzer class
|
141
|
+
# @param ar analyzer results
|
142
|
+
# @param val value to be analyzed
|
143
|
+
def update_numeric_results(ac, ar, val)
|
144
|
+
cval = ac.convert(val)
|
145
|
+
ar[:min] = cval if ar[:min].nil? || cval < ar[:min]
|
146
|
+
ar[:max] = cval if ar[:max].nil? || cval > ar[:max]
|
147
|
+
end
|
148
|
+
|
149
|
+
def update_results(analyzer, res, val)
|
150
|
+
ac = analyzer[:class]
|
151
|
+
ar = analyzer[:results]
|
152
|
+
ar[:count] += 1
|
153
|
+
|
154
|
+
update_numeric_results(ac, ar, val) if res && ac.numeric?
|
89
155
|
end
|
90
156
|
end
|
91
157
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
5
|
+
module Csv2Psql
|
6
|
+
module Analyzers
|
7
|
+
# BaseAnalyzer value matcher
|
8
|
+
class BaseAnalyzer
|
9
|
+
class << self
|
10
|
+
def analyze(_val)
|
11
|
+
nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def convert(val)
|
15
|
+
val
|
16
|
+
end
|
17
|
+
|
18
|
+
def numeric?
|
19
|
+
const_get('CLASS') == :numeric
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def analyze(val)
|
24
|
+
self.class.analyze(val)
|
25
|
+
end
|
26
|
+
|
27
|
+
def convert(val)
|
28
|
+
self.class.convert(val)
|
29
|
+
end
|
30
|
+
|
31
|
+
def numeric?
|
32
|
+
self.class.numeric?
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -1,43 +1,23 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# Bigint value matcher
|
6
|
-
class Bigint
|
8
|
+
class Bigint < BaseAnalyzer
|
7
9
|
TYPE = :bigint
|
8
10
|
CLASS = :numeric
|
9
11
|
WEIGHT = 4
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
@min = nil
|
16
|
-
@max = nil
|
17
|
-
end
|
18
|
-
|
19
|
-
def analyze(val)
|
20
|
-
return unless val.is_a?(Integer) || (val && val.match(/^\d+$/))
|
21
|
-
|
22
|
-
update(convert(val))
|
23
|
-
end
|
24
|
-
|
25
|
-
def convert(val)
|
26
|
-
val.to_i
|
27
|
-
end
|
28
|
-
|
29
|
-
def to_h
|
30
|
-
{
|
31
|
-
count: @count,
|
32
|
-
min: @min,
|
33
|
-
max: @max
|
34
|
-
}
|
35
|
-
end
|
13
|
+
class << self
|
14
|
+
def analyze(val)
|
15
|
+
val.is_a?(Integer) || (val && !val.match(/^\d+$/).nil?)
|
16
|
+
end
|
36
17
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
@max = val if @max.nil? || val > @max
|
18
|
+
def convert(val)
|
19
|
+
val.to_i
|
20
|
+
end
|
41
21
|
end
|
42
22
|
end
|
43
23
|
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
5
|
+
module Csv2Psql
|
6
|
+
module Analyzers
|
7
|
+
# Bolean value matcher
|
8
|
+
class Boolean < BaseAnalyzer
|
9
|
+
TYPE = :boolean
|
10
|
+
CLASS = :boolean
|
11
|
+
WEIGHT = 5
|
12
|
+
|
13
|
+
BOOLEAN_VALUES = %w(true false 0 1)
|
14
|
+
BOOLEAN_VALUES_MAP = {}
|
15
|
+
BOOLEAN_VALUES.each do |k|
|
16
|
+
BOOLEAN_VALUES_MAP[k] = true
|
17
|
+
end
|
18
|
+
|
19
|
+
class << self
|
20
|
+
def analyze(val)
|
21
|
+
return if val.nil? || val.empty?
|
22
|
+
BOOLEAN_VALUES_MAP.key?(val.downcase)
|
23
|
+
end
|
24
|
+
|
25
|
+
def convert(val)
|
26
|
+
val.to_i
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -1,29 +1,19 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# Character value matcher
|
6
|
-
class Character
|
8
|
+
class Character < BaseAnalyzer
|
7
9
|
TYPE = :bigint
|
8
10
|
CLASS = :character
|
9
11
|
WEIGHT = 2
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
def analyze(val)
|
18
|
-
match = val && val.to_s.length == 1
|
19
|
-
return unless match
|
20
|
-
@count += 1
|
21
|
-
end
|
22
|
-
|
23
|
-
def to_h
|
24
|
-
{
|
25
|
-
count: @count
|
26
|
-
}
|
13
|
+
class << self
|
14
|
+
def analyze(val)
|
15
|
+
val && val.to_s.length == 1
|
16
|
+
end
|
27
17
|
end
|
28
18
|
end
|
29
19
|
end
|
@@ -1,43 +1,26 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# Decimal value matcher
|
6
|
-
class Decimal
|
8
|
+
class Decimal < BaseAnalyzer
|
7
9
|
TYPE = :decimal
|
8
10
|
CLASS = :numeric
|
9
11
|
WEIGHT = 3
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
update(convert(val))
|
23
|
-
end
|
24
|
-
|
25
|
-
def convert(val)
|
26
|
-
val.to_f
|
27
|
-
end
|
28
|
-
|
29
|
-
def to_h
|
30
|
-
{
|
31
|
-
count: @count,
|
32
|
-
min: @min,
|
33
|
-
max: @max
|
34
|
-
}
|
35
|
-
end
|
36
|
-
|
37
|
-
def update(val)
|
38
|
-
@count += 1
|
39
|
-
@min = val if @min.nil? || val < @min
|
40
|
-
@max = val if @max.nil? || val > @max
|
12
|
+
RE = /^[-+]?[0-9]*[.,]?[0-9]+([eE][-+]?[0-9]+)?$/
|
13
|
+
|
14
|
+
class << self
|
15
|
+
def analyze(val)
|
16
|
+
return true if val.is_a?(Float)
|
17
|
+
res = val && val.match(RE)
|
18
|
+
!res.nil?
|
19
|
+
end
|
20
|
+
|
21
|
+
def convert(val)
|
22
|
+
val.to_f
|
23
|
+
end
|
41
24
|
end
|
42
25
|
end
|
43
26
|
end
|
@@ -1,29 +1,19 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# Null value matcher
|
6
|
-
class Null
|
8
|
+
class Null < BaseAnalyzer
|
7
9
|
TYPE = :null
|
8
10
|
CLASS = nil # TODO: Maybe use better class for Null type?
|
9
11
|
WEIGHT = 0
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
end
|
16
|
-
|
17
|
-
def analyze(val)
|
18
|
-
match = val.nil? || val.empty?
|
19
|
-
return unless match
|
20
|
-
@count += 1
|
21
|
-
end
|
22
|
-
|
23
|
-
def to_h
|
24
|
-
{
|
25
|
-
count: @count
|
26
|
-
}
|
13
|
+
class << self
|
14
|
+
def analyze(val)
|
15
|
+
val.nil? || val.empty?
|
16
|
+
end
|
27
17
|
end
|
28
18
|
end
|
29
19
|
end
|
@@ -1,36 +1,19 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# UUID value matcher
|
6
|
-
class String
|
8
|
+
class String < BaseAnalyzer
|
7
9
|
TYPE = :string
|
8
10
|
CLASS = :character
|
9
11
|
WEIGHT = 1
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
@min = nil
|
16
|
-
@max = nil
|
17
|
-
end
|
18
|
-
|
19
|
-
def analyze(val)
|
20
|
-
match = val.is_a?(::String)
|
21
|
-
return unless match
|
22
|
-
len = val.length
|
23
|
-
@min = len if @min.nil? || len < @min
|
24
|
-
@max = len if @max.nil? || len > @max
|
25
|
-
@count += 1
|
26
|
-
end
|
27
|
-
|
28
|
-
def to_h
|
29
|
-
{
|
30
|
-
count: @count,
|
31
|
-
min: @min,
|
32
|
-
max: @max
|
33
|
-
}
|
13
|
+
class << self
|
14
|
+
def analyze(val)
|
15
|
+
val.is_a?(::String)
|
16
|
+
end
|
34
17
|
end
|
35
18
|
end
|
36
19
|
end
|
@@ -1,31 +1,22 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
+
require_relative 'base_analyzer'
|
4
|
+
|
3
5
|
module Csv2Psql
|
4
6
|
module Analyzers
|
5
7
|
# UUID value matcher
|
6
|
-
class Uuid
|
8
|
+
class Uuid < BaseAnalyzer
|
7
9
|
TYPE = :uuid
|
8
10
|
CLASS = :uuid
|
9
11
|
WEIGHT = 5
|
10
12
|
|
11
|
-
RE =
|
12
|
-
|
13
|
-
attr_reader :count
|
14
|
-
|
15
|
-
def initialize
|
16
|
-
@count = 0
|
17
|
-
end
|
18
|
-
|
19
|
-
def analyze(val)
|
20
|
-
match = val && val.match(RE)
|
21
|
-
return if match.nil?
|
22
|
-
@count += 1
|
23
|
-
end
|
13
|
+
RE = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}$/ # rubocop:disable Metrics/LineLength
|
24
14
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
15
|
+
class << self
|
16
|
+
def analyze(val)
|
17
|
+
match = val && val.match(RE)
|
18
|
+
!match.nil?
|
19
|
+
end
|
29
20
|
end
|
30
21
|
end
|
31
22
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'lru'
|
4
|
+
|
5
|
+
module Csv2Psql
|
6
|
+
# Last Recently Used cache implementation
|
7
|
+
class Cache
|
8
|
+
attr_accessor :max_size
|
9
|
+
|
10
|
+
def initialize(max_size = 1000)
|
11
|
+
@cache = ::Cache::LRU.new(max_elements: max_size)
|
12
|
+
end
|
13
|
+
|
14
|
+
def put(key, value)
|
15
|
+
@cache.put(key, value)
|
16
|
+
end
|
17
|
+
|
18
|
+
def get(key, &block)
|
19
|
+
@cache.get(key, &block)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/csv2psql/cli/app.rb
CHANGED
@@ -20,13 +20,13 @@ program_desc "csv2psql #{Csv2Psql::VERSION} (Codename: #{Csv2Psql::CODENAME})"
|
|
20
20
|
cmds = {
|
21
21
|
h: {
|
22
22
|
desc: 'Header row included',
|
23
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
23
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['header']
|
24
24
|
},
|
25
25
|
|
26
26
|
d: {
|
27
27
|
desc: 'Column delimiter',
|
28
28
|
type: String,
|
29
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
29
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['delimiter']
|
30
30
|
},
|
31
31
|
|
32
32
|
l: {
|
@@ -38,13 +38,13 @@ cmds = {
|
|
38
38
|
q: {
|
39
39
|
desc: 'Quoting character',
|
40
40
|
type: String,
|
41
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
41
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['quote']
|
42
42
|
},
|
43
43
|
|
44
44
|
s: {
|
45
45
|
desc: 'Line separator',
|
46
46
|
type: String,
|
47
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
47
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['separator']
|
48
48
|
},
|
49
49
|
|
50
50
|
'skip' => {
|
@@ -17,7 +17,7 @@ Csv2Psql::Cli.module_eval do
|
|
17
17
|
res.files.each do |_fname, results|
|
18
18
|
results[:columns].each do |_k, v|
|
19
19
|
v.each do |d, det|
|
20
|
-
v[d] = det
|
20
|
+
v[d] = det[:results]
|
21
21
|
end
|
22
22
|
end
|
23
23
|
end
|
@@ -30,7 +30,7 @@ Csv2Psql::Cli.module_eval do
|
|
30
30
|
header = ['column'] + res.analyzers.map { |a| a[:name] }
|
31
31
|
|
32
32
|
rows = details[:columns].map do |k, v|
|
33
|
-
[k] + v.keys.map { |name| v[name]
|
33
|
+
[k] + v.keys.map { |name| v[name][:results][:count] }
|
34
34
|
end
|
35
35
|
|
36
36
|
Terminal::Table.new title: file, headings: header, rows: rows
|
data/lib/csv2psql/cli/shared.rb
CHANGED
@@ -13,13 +13,13 @@ Csv2Psql::Cli.module_eval do
|
|
13
13
|
cmds = {
|
14
14
|
h: {
|
15
15
|
desc: 'Header row included',
|
16
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
16
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['header']
|
17
17
|
},
|
18
18
|
|
19
19
|
d: {
|
20
20
|
desc: 'Column delimiter',
|
21
21
|
type: String,
|
22
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
22
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['delimiter']
|
23
23
|
},
|
24
24
|
|
25
25
|
l: {
|
@@ -31,13 +31,13 @@ Csv2Psql::Cli.module_eval do
|
|
31
31
|
q: {
|
32
32
|
desc: 'Quoting character',
|
33
33
|
type: String,
|
34
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
34
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['quote']
|
35
35
|
},
|
36
36
|
|
37
37
|
s: {
|
38
38
|
desc: 'Line separator',
|
39
39
|
type: String,
|
40
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[
|
40
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['separator']
|
41
41
|
},
|
42
42
|
|
43
43
|
'skip' => {
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'multi_json'
|
4
|
+
|
5
|
+
require_relative '../helpers/json_helper'
|
6
|
+
|
7
|
+
module Csv2Psql
|
8
|
+
# Configuration module
|
9
|
+
module Config
|
10
|
+
BASE_DIR = File.join(File.dirname(__FILE__), '..', '..', '..')
|
11
|
+
CONFIG_PATH = File.join(BASE_DIR, 'config', 'config.json')
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def config(path = CONFIG_PATH)
|
15
|
+
@config ||= load_config(path)
|
16
|
+
@config
|
17
|
+
end
|
18
|
+
|
19
|
+
def load_config(path = CONFIG_PATH)
|
20
|
+
JsonHelper.load_file(path)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'csv'
|
4
|
+
|
5
|
+
require_relative 'base'
|
6
|
+
|
7
|
+
module Csv2Psql
|
8
|
+
# Frontend parsers
|
9
|
+
module Frontend
|
10
|
+
# Csv frontend class
|
11
|
+
class Csv < Base
|
12
|
+
def open(path, open_opts = 'rt', csv_opts = {}, &_block)
|
13
|
+
CSV.open(path, open_opts, csv_opts) do |csv|
|
14
|
+
Proc.new.call(csv)
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
File without changes
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
require 'multi_json'
|
4
|
+
|
5
|
+
module Csv2Psql
|
6
|
+
# Json Helper
|
7
|
+
class JsonHelper
|
8
|
+
BASE_DIR = File.join(File.dirname(__FILE__), '..')
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def load_file(path)
|
12
|
+
# Load input file
|
13
|
+
raw = IO.read(path)
|
14
|
+
|
15
|
+
# Try to parse json from loaded data
|
16
|
+
begin
|
17
|
+
return MultiJson.load(raw)
|
18
|
+
rescue Exception => e # rubocop:disable RescueException
|
19
|
+
log_exception(e)
|
20
|
+
raise e
|
21
|
+
end
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
|
25
|
+
def log_exception(e)
|
26
|
+
puts 'Invalid json, see error.txt'
|
27
|
+
File.open('error.txt', 'wt') { |f| f.write(e.to_s) }
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -1,12 +1,14 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
3
|
require 'csv'
|
4
|
-
require 'multi_json'
|
5
4
|
require 'pathname'
|
6
5
|
require 'pp'
|
7
6
|
|
8
7
|
require_relative '../analyzer/analyzer'
|
8
|
+
require_relative '../cache/cache'
|
9
|
+
require_relative '../frontend/csv'
|
9
10
|
require_relative '../generator/generator'
|
11
|
+
require_relative '../helpers/config_helper'
|
10
12
|
require_relative '../helpers/csv_helper'
|
11
13
|
require_relative '../helpers/erb_helper'
|
12
14
|
require_relative '../output/output'
|
@@ -17,18 +19,14 @@ module Csv2Psql
|
|
17
19
|
class Processor
|
18
20
|
attr_reader :analyzer, :generator, :output, :path
|
19
21
|
|
20
|
-
DEFAULT_OPTIONS =
|
21
|
-
delimiter: ',',
|
22
|
-
header: true,
|
23
|
-
separator: :auto,
|
24
|
-
transaction: false,
|
25
|
-
quote: '"'
|
26
|
-
}
|
22
|
+
DEFAULT_OPTIONS = ConfigHelper.config['processor']
|
27
23
|
|
28
24
|
def initialize
|
29
25
|
@output = Output.new
|
30
26
|
@generator = Generator.new(@output)
|
31
|
-
@
|
27
|
+
@cache = Cache.new
|
28
|
+
@analyzer = Analyzer.new(@cache)
|
29
|
+
@frontend = Frontend::Csv.new
|
32
30
|
end
|
33
31
|
|
34
32
|
def analyze(paths, opts = {})
|
@@ -73,23 +71,36 @@ module Csv2Psql
|
|
73
71
|
end
|
74
72
|
|
75
73
|
def merge_csv_options(opts = {})
|
76
|
-
header = !opts[
|
77
|
-
{
|
78
|
-
col_sep: opts[:delimiter] || DEFAULT_OPTIONS[:delimiter],
|
74
|
+
header = !opts['header'].nil? ? opts['header'] : DEFAULT_OPTIONS['header']
|
75
|
+
res = {
|
79
76
|
headers: header,
|
80
|
-
|
81
|
-
quote_char: opts[:quote] || DEFAULT_OPTIONS[:quote]
|
77
|
+
quote_char: opts['quote'] || DEFAULT_OPTIONS['quote']
|
82
78
|
}
|
79
|
+
res[:col_sep] = opts['delimiter'] if opts['delimiter']
|
80
|
+
res[:row_sep] = opts['separator'] if opts['separator']
|
81
|
+
res
|
82
|
+
end
|
83
|
+
|
84
|
+
def process_file(path, csv, opts, &block)
|
85
|
+
lines = 0
|
86
|
+
limit = opts[:l]
|
87
|
+
skip = opts[:skip]
|
88
|
+
csv.each do |row|
|
89
|
+
lines += 1
|
90
|
+
next if skip > 0 && lines <= skip
|
91
|
+
|
92
|
+
with_row(path, row, opts, &block)
|
93
|
+
|
94
|
+
return if limit > 0 && lines >= limit
|
95
|
+
end
|
83
96
|
end
|
84
97
|
|
85
98
|
def with_path(path, opts = {}, &block)
|
86
99
|
output.write 'BEGIN;' if opts[:transaction]
|
87
100
|
csv_opts = merge_csv_options(opts)
|
88
101
|
@first_row = true
|
89
|
-
|
90
|
-
csv
|
91
|
-
with_row(path, row, opts, &block)
|
92
|
-
end
|
102
|
+
@frontend.open(path, 'rt', csv_opts) do |csv|
|
103
|
+
process_file(path, csv, opts, &block)
|
93
104
|
end
|
94
105
|
output.write 'COMMIT;' if opts[:transaction]
|
95
106
|
end
|
@@ -101,9 +112,9 @@ module Csv2Psql
|
|
101
112
|
end
|
102
113
|
end
|
103
114
|
|
104
|
-
def with_row(path, row, _opts = {}, &
|
115
|
+
def with_row(path, row, _opts = {}, &_block)
|
105
116
|
args = { path: path, row: row }
|
106
|
-
|
117
|
+
Proc.new.call(args)
|
107
118
|
end
|
108
119
|
end
|
109
120
|
end
|
data/lib/csv2psql/version.rb
CHANGED
data/spec/helpers/cli_helper.rb
CHANGED
@@ -7,11 +7,11 @@ module CliHelper
|
|
7
7
|
# Execute block and capture its stdou
|
8
8
|
# @param block Block to be executed with stdout redirected
|
9
9
|
# @returns Captured output as string
|
10
|
-
def capture_stdout(&
|
10
|
+
def capture_stdout(&_block)
|
11
11
|
original_stdout = $stdout
|
12
12
|
$stdout = fake = StringIO.new
|
13
13
|
begin
|
14
|
-
|
14
|
+
Proc.new.call if block_given?
|
15
15
|
ensure
|
16
16
|
$stdout = original_stdout
|
17
17
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv2psql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomas Korcak
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-09-
|
11
|
+
date: 2014-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gli
|
@@ -44,6 +44,26 @@ dependencies:
|
|
44
44
|
- - "~>"
|
45
45
|
- !ruby/object:Gem::Version
|
46
46
|
version: 1.8.1
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: lru
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
requirements:
|
51
|
+
- - "~>"
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0.1'
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: 0.1.0
|
57
|
+
type: :runtime
|
58
|
+
prerelease: false
|
59
|
+
version_requirements: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - "~>"
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '0.1'
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 0.1.0
|
47
67
|
- !ruby/object:Gem::Dependency
|
48
68
|
name: multi_json
|
49
69
|
requirement: !ruby/object:Gem::Requirement
|
@@ -244,29 +264,40 @@ files:
|
|
244
264
|
- Rakefile
|
245
265
|
- TODO.md
|
246
266
|
- bin/csv2psql
|
267
|
+
- config/config.json
|
247
268
|
- data/cia-data-all.csv
|
248
269
|
- data/sample.csv
|
270
|
+
- data/sample_bool.csv
|
249
271
|
- data/sample_semicolons.csv
|
250
272
|
- lib/csv2psql.rb
|
251
273
|
- lib/csv2psql/analyzer/analyzer.rb
|
274
|
+
- lib/csv2psql/analyzer/types/base_analyzer.rb
|
252
275
|
- lib/csv2psql/analyzer/types/bigint.rb
|
276
|
+
- lib/csv2psql/analyzer/types/boolean.rb
|
253
277
|
- lib/csv2psql/analyzer/types/character.rb
|
254
278
|
- lib/csv2psql/analyzer/types/decimal.rb
|
255
279
|
- lib/csv2psql/analyzer/types/null.rb
|
256
280
|
- lib/csv2psql/analyzer/types/string.rb
|
257
281
|
- lib/csv2psql/analyzer/types/uuid.rb
|
282
|
+
- lib/csv2psql/cache/cache.rb
|
258
283
|
- lib/csv2psql/cli/app.rb
|
259
284
|
- lib/csv2psql/cli/cli.rb
|
260
285
|
- lib/csv2psql/cli/cmd/analyze_cmd.rb
|
261
286
|
- lib/csv2psql/cli/cmd/convert_cmd.rb
|
262
287
|
- lib/csv2psql/cli/cmd/version_cmd.rb
|
263
288
|
- lib/csv2psql/cli/shared.rb
|
289
|
+
- lib/csv2psql/config/config.rb
|
264
290
|
- lib/csv2psql/convert/convert.rb
|
265
|
-
- lib/csv2psql/dialects/psql.rb
|
266
291
|
- lib/csv2psql/extensions/string.rb
|
292
|
+
- lib/csv2psql/frontend/base.rb
|
293
|
+
- lib/csv2psql/frontend/csv.rb
|
294
|
+
- lib/csv2psql/frontend/frontend.rb
|
295
|
+
- lib/csv2psql/generator/dialects/psql.rb
|
267
296
|
- lib/csv2psql/generator/generator.rb
|
297
|
+
- lib/csv2psql/helpers/config_helper.rb
|
268
298
|
- lib/csv2psql/helpers/csv_helper.rb
|
269
299
|
- lib/csv2psql/helpers/erb_helper.rb
|
300
|
+
- lib/csv2psql/helpers/json_helper.rb
|
270
301
|
- lib/csv2psql/lib.rb
|
271
302
|
- lib/csv2psql/output/output.rb
|
272
303
|
- lib/csv2psql/processor/processor.rb
|