csv2psql 0.0.10 → 0.0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 017e866dc4a84a448f04c6d360ee95d215b8e6bc
4
- data.tar.gz: 036e31c4b92508fb0d66942779378c8b11f46d94
3
+ metadata.gz: 5299e63f4ab0f21575bc04d1b0f4e863f3121197
4
+ data.tar.gz: 32532a0c828576f90c016f51a880756c4161ee71
5
5
  SHA512:
6
- metadata.gz: 6910eb4b0578c3a5699b494673e61f3dec5c1f98c0fe503e359e27e4804d8739258ec0fc8db6effed1df08728bae21b0cec63d02caccb6d1cea2933602e459ad
7
- data.tar.gz: dc146c73e70760956c91a7b947eac86fb3d1f212f997c117ddea8b3a2b22be6ae402d15f1400107b565411c946235e6f86598e1fe5dccea6cf3c47fa92ee5c79
6
+ metadata.gz: b227bf6c07fff2c0beafe9bfdba44654aaf372c2852ebf4d7430ac95cbd5478fc7cf2d4c0b26d0a168f551bd753f8997ebf17d54b8b59896b3614fe1df9cc394
7
+ data.tar.gz: 41595c98914f724d354b530e5fa9a21082d3d03b490e99e6cfb5b1efe6c60be1214b164f38403353d448bf42cc67bdf8a9fe93dde3ee4c5f955d9c3afb9b253b
data/.gemspec CHANGED
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
25
25
 
26
26
  s.add_dependency 'gli', '~> 2.11', '>= 2.11.0'
27
27
  s.add_dependency 'json_pure', '~> 1.8.1'
28
+ s.add_dependency 'lru', '~> 0.1', '>= 0.1.0'
28
29
  s.add_dependency 'multi_json', '~> 1.10.0'
29
30
  s.add_dependency 'rake', '~> 10.3', '>= 10.3.2'
30
31
  s.add_dependency 'terminal-table', '~> 1.4', '>= 1.4.5'
@@ -0,0 +1,8 @@
1
+ {
2
+ "processor": {
3
+ "delimiter": ",",
4
+ "header": true,
5
+ "transaction": false,
6
+ "quote": "\""
7
+ }
8
+ }
@@ -0,0 +1,13 @@
1
+ id,Firstname,Lastname,Address.Street,Address.City,Address.Details.Note,Married
2
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,True
3
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,False
4
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,1
5
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,0
6
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,blah
7
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,no
8
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"0"
9
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"1"
10
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"True"
11
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"False"
12
+ 12345,Joe,Doe,"#2140 Taylor Street, 94133",San Francisco,Pool available,"blah"
13
+ 45678,Jack,Plumber,"#111 Sutter St, 94104",San Francisco,Korean Deli near to main entrance,"no"
@@ -1,7 +1,5 @@
1
1
  # encoding: UTF-8
2
2
 
3
- require 'csv'
4
- require 'multi_json'
5
3
  require 'pathname'
6
4
  require 'pp'
7
5
 
@@ -10,14 +8,16 @@ require_relative '../extensions/string'
10
8
 
11
9
  module Csv2Psql
12
10
  # Analyzer file analyzer class
13
- class Analyzer
11
+ class Analyzer # rubocop:disable Metrics/ClassLength
14
12
  DEFAULT_OPTIONS = {}
15
13
  ANALYZERS_DIR = File.join(File.dirname(__FILE__), 'types')
14
+ EXCLUDED_ANALYZERS = ['base_analyzer']
16
15
 
17
16
  attr_reader :analyzers, :files
18
17
 
19
- def initialize
18
+ def initialize(cache = nil)
20
19
  @files = {}
20
+ @cache = cache
21
21
  @analyzers = load_analyzers
22
22
  end
23
23
 
@@ -25,23 +25,52 @@ module Csv2Psql
25
25
  data = get_data(path)
26
26
 
27
27
  header = CsvHelper.get_header(row, opts)
28
+ analyze_row(header, row, data)
29
+
30
+ data[:lines] = data[:lines] + 1
31
+ end
32
+
33
+ def analyze_column(analyzer, val, opts = { use_cache: false })
34
+ if opts[:use_cache]
35
+ res = cached_result(val) do
36
+ analyzer[:class].analyze(val)
37
+ end
38
+ else
39
+ res = analyzer[:class].analyze(val)
40
+ end
41
+
42
+ update_results(analyzer, res, val) if res
43
+ end
44
+
45
+ def analyze_row(header, row, data)
28
46
  header.each do |h|
29
47
  col = get_column(data, h)
30
- val = row[h]
31
48
  col.each do |_name, analyzer|
32
- analyzer.analyze(val)
49
+ analyze_column(analyzer, row[h])
33
50
  end
34
51
  end
52
+ end
35
53
 
36
- data[:lines] = data[:lines] + 1
54
+ def cached_result(val, &_block)
55
+ res = @cache.get(val)
56
+ if res.nil?
57
+ res = Proc.new.call(val)
58
+ @cache.put(val, res)
59
+ end
60
+ res
37
61
  end
38
62
 
63
+ # Create column analyzers
39
64
  def create_column(data, column)
40
65
  data[:columns][column] = {}
41
66
  res = data[:columns][column]
42
67
 
43
68
  analyzers.each do |analyzer|
44
- res[analyzer[:name]] = analyzer[:class].new
69
+ analyzer_class = analyzer[:class]
70
+ res[analyzer[:name]] = {
71
+ class: analyzer_class.new,
72
+ results: create_results(analyzer_class)
73
+ }
45
74
  end
46
75
 
47
76
  res
@@ -56,6 +85,17 @@ module Csv2Psql
56
85
  files[path]
57
86
  end
58
87
 
88
+ def create_results(analyzer_class)
89
+ res = {
90
+ count: 0
91
+ }
92
+
93
+ res[:min] = nil if analyzer_class.numeric?
94
+ res[:max] = nil if analyzer_class.numeric?
95
+
96
+ res
97
+ end
98
+
59
99
  def get_data(path)
60
100
  return files[path] if files.key?(path)
61
101
 
@@ -69,23 +109,49 @@ module Csv2Psql
69
109
  create_column(data, column)
70
110
  end
71
111
 
72
- def load_analyze_class(analyzer_class)
112
+ def load_analyzer_class(analyzer_class)
73
113
  Object.const_get('Csv2Psql')
74
- .const_get('Analyzers')
75
- .const_get(analyzer_class)
114
+ .const_get('Analyzers')
115
+ .const_get(analyzer_class)
116
+ end
117
+
118
+ def load_analyzer(path)
119
+ fname = File.basename(path, '.rb')
120
+ analyzer_class = fname.camel_case
121
+ require(path)
122
+
123
+ {
124
+ name: analyzer_class,
125
+ class: load_analyzer_class(analyzer_class)
126
+ }
76
127
  end
77
128
 
78
129
  def load_analyzers
79
- Dir[ANALYZERS_DIR + '**/*.rb'].map do |path|
80
- fname = File.basename(path, '.rb')
81
- analyzer_class = fname.camel_case
82
- require(path)
83
-
84
- {
85
- name: analyzer_class,
86
- class: load_analyze_class(analyzer_class)
87
- }
130
+ res = Dir[ANALYZERS_DIR + '**/*.rb'].map do |path|
131
+ name = File.basename(path, '.rb')
132
+ next if EXCLUDED_ANALYZERS.include?(name)
133
+ load_analyzer(path)
88
134
  end
135
+
136
+ res.compact
137
+ end
138
+
139
+ # Update numeric results
140
+ # @param ac analyzer class
141
+ # @param ar analyzer results
142
+ # @param val value to be analyzed
143
+ def update_numeric_results(ac, ar, val)
144
+ cval = ac.convert(val)
145
+ ar[:min] = cval if ar[:min].nil? || cval < ar[:min]
146
+ ar[:max] = cval if ar[:max].nil? || cval > ar[:max]
147
+ end
148
+
149
+ def update_results(analyzer, res, val)
150
+ ac = analyzer[:class]
151
+ ar = analyzer[:results]
152
+ ar[:count] += 1
153
+
154
+ update_numeric_results(ac, ar, val) if res && ac.numeric?
89
155
  end
90
156
  end
91
157
  end
@@ -0,0 +1,36 @@
1
+ # encoding: UTF-8
2
+
3
+ require_relative 'base_analyzer'
4
+
5
+ module Csv2Psql
6
+ module Analyzers
7
+ # BaseAnalyzer value matcher
8
+ class BaseAnalyzer
9
+ class << self
10
+ def analyze(_val)
11
+ nil
12
+ end
13
+
14
+ def convert(val)
15
+ val
16
+ end
17
+
18
+ def numeric?
19
+ const_get('CLASS') == :numeric
20
+ end
21
+ end
22
+
23
+ def analyze(val)
24
+ self.class.analyze(val)
25
+ end
26
+
27
+ def convert(val)
28
+ self.class.convert(val)
29
+ end
30
+
31
+ def numeric?
32
+ self.class.numeric?
33
+ end
34
+ end
35
+ end
36
+ end
@@ -1,43 +1,23 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # Bigint value matcher
6
- class Bigint
8
+ class Bigint < BaseAnalyzer
7
9
  TYPE = :bigint
8
10
  CLASS = :numeric
9
11
  WEIGHT = 4
10
12
 
11
- attr_reader :count, :min, :max
12
-
13
- def initialize
14
- @count = 0
15
- @min = nil
16
- @max = nil
17
- end
18
-
19
- def analyze(val)
20
- return unless val.is_a?(Integer) || (val && val.match(/^\d+$/))
21
-
22
- update(convert(val))
23
- end
24
-
25
- def convert(val)
26
- val.to_i
27
- end
28
-
29
- def to_h
30
- {
31
- count: @count,
32
- min: @min,
33
- max: @max
34
- }
35
- end
13
+ class << self
14
+ def analyze(val)
15
+ val.is_a?(Integer) || (val && !val.match(/^\d+$/).nil?)
16
+ end
36
17
 
37
- def update(val)
38
- @count += 1
39
- @min = val if @min.nil? || val < @min
40
- @max = val if @max.nil? || val > @max
18
+ def convert(val)
19
+ val.to_i
20
+ end
41
21
  end
42
22
  end
43
23
  end
@@ -0,0 +1,31 @@
1
+ # encoding: UTF-8
2
+
3
+ require_relative 'base_analyzer'
4
+
5
+ module Csv2Psql
6
+ module Analyzers
7
+ # Bolean value matcher
8
+ class Boolean < BaseAnalyzer
9
+ TYPE = :boolean
10
+ CLASS = :boolean
11
+ WEIGHT = 5
12
+
13
+ BOOLEAN_VALUES = %w(true false 0 1)
14
+ BOOLEAN_VALUES_MAP = {}
15
+ BOOLEAN_VALUES.each do |k|
16
+ BOOLEAN_VALUES_MAP[k] = true
17
+ end
18
+
19
+ class << self
20
+ def analyze(val)
21
+ return if val.nil? || val.empty?
22
+ BOOLEAN_VALUES_MAP.key?(val.downcase)
23
+ end
24
+
25
+ def convert(val)
26
+ val.to_i
27
+ end
28
+ end
29
+ end
30
+ end
31
+ end
@@ -1,29 +1,19 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # Character value matcher
6
- class Character
8
+ class Character < BaseAnalyzer
7
9
  TYPE = :bigint
8
10
  CLASS = :character
9
11
  WEIGHT = 2
10
12
 
11
- attr_reader :count
12
-
13
- def initialize
14
- @count = 0
15
- end
16
-
17
- def analyze(val)
18
- match = val && val.to_s.length == 1
19
- return unless match
20
- @count += 1
21
- end
22
-
23
- def to_h
24
- {
25
- count: @count
26
- }
13
+ class << self
14
+ def analyze(val)
15
+ val && val.to_s.length == 1
16
+ end
27
17
  end
28
18
  end
29
19
  end
@@ -1,43 +1,26 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # Decimal value matcher
6
- class Decimal
8
+ class Decimal < BaseAnalyzer
7
9
  TYPE = :decimal
8
10
  CLASS = :numeric
9
11
  WEIGHT = 3
10
-
11
- attr_reader :count, :min, :max
12
-
13
- def initialize
14
- @count = 0
15
- @min = nil
16
- @max = nil
17
- end
18
-
19
- def analyze(val)
20
- return unless val.is_a?(Float) || (val && val.match(/(\d+[,.]\d+)/))
21
-
22
- update(convert(val))
23
- end
24
-
25
- def convert(val)
26
- val.to_f
27
- end
28
-
29
- def to_h
30
- {
31
- count: @count,
32
- min: @min,
33
- max: @max
34
- }
35
- end
36
-
37
- def update(val)
38
- @count += 1
39
- @min = val if @min.nil? || val < @min
40
- @max = val if @max.nil? || val > @max
12
+ RE = /^[-+]?[0-9]*[.,]?[0-9]+([eE][-+]?[0-9]+)?$/
13
+
14
+ class << self
15
+ def analyze(val)
16
+ return true if val.is_a?(Float)
17
+ res = val && val.match(RE)
18
+ !res.nil?
19
+ end
20
+
21
+ def convert(val)
22
+ val.to_f
23
+ end
41
24
  end
42
25
  end
43
26
  end
@@ -1,29 +1,19 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # Null value matcher
6
- class Null
8
+ class Null < BaseAnalyzer
7
9
  TYPE = :null
8
10
  CLASS = nil # TODO: Maybe use better class for Null type?
9
11
  WEIGHT = 0
10
12
 
11
- attr_reader :count
12
-
13
- def initialize
14
- @count = 0
15
- end
16
-
17
- def analyze(val)
18
- match = val.nil? || val.empty?
19
- return unless match
20
- @count += 1
21
- end
22
-
23
- def to_h
24
- {
25
- count: @count
26
- }
13
+ class << self
14
+ def analyze(val)
15
+ val.nil? || val.empty?
16
+ end
27
17
  end
28
18
  end
29
19
  end
@@ -1,36 +1,19 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # UUID value matcher
6
- class String
8
+ class String < BaseAnalyzer
7
9
  TYPE = :string
8
10
  CLASS = :character
9
11
  WEIGHT = 1
10
12
 
11
- attr_reader :count, :min, :max
12
-
13
- def initialize
14
- @count = 0
15
- @min = nil
16
- @max = nil
17
- end
18
-
19
- def analyze(val)
20
- match = val.is_a?(::String)
21
- return unless match
22
- len = val.length
23
- @min = len if @min.nil? || len < @min
24
- @max = len if @max.nil? || len > @max
25
- @count += 1
26
- end
27
-
28
- def to_h
29
- {
30
- count: @count,
31
- min: @min,
32
- max: @max
33
- }
13
+ class << self
14
+ def analyze(val)
15
+ val.is_a?(::String)
16
+ end
34
17
  end
35
18
  end
36
19
  end
@@ -1,31 +1,22 @@
1
1
  # encoding: UTF-8
2
2
 
3
+ require_relative 'base_analyzer'
4
+
3
5
  module Csv2Psql
4
6
  module Analyzers
5
7
  # UUID value matcher
6
- class Uuid
8
+ class Uuid < BaseAnalyzer
7
9
  TYPE = :uuid
8
10
  CLASS = :uuid
9
11
  WEIGHT = 5
10
12
 
11
- RE = /[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}/ # rubocop:disable Metrics/LineLength
12
-
13
- attr_reader :count
14
-
15
- def initialize
16
- @count = 0
17
- end
18
-
19
- def analyze(val)
20
- match = val && val.match(RE)
21
- return if match.nil?
22
- @count += 1
23
- end
13
+ RE = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}$/ # rubocop:disable Metrics/LineLength
24
14
 
25
- def to_h
26
- {
27
- count: @count
28
- }
15
+ class << self
16
+ def analyze(val)
17
+ match = val && val.match(RE)
18
+ !match.nil?
19
+ end
29
20
  end
30
21
  end
31
22
  end
@@ -0,0 +1,22 @@
1
+ # encoding: UTF-8
2
+
3
+ require 'lru'
4
+
5
+ module Csv2Psql
6
+ # Last Recently Used cache implementation
7
+ class Cache
8
+ attr_accessor :max_size
9
+
10
+ def initialize(max_size = 1000)
11
+ @cache = ::Cache::LRU.new(max_elements: max_size)
12
+ end
13
+
14
+ def put(key, value)
15
+ @cache.put(key, value)
16
+ end
17
+
18
+ def get(key, &block)
19
+ @cache.get(key, &block)
20
+ end
21
+ end
22
+ end
@@ -20,13 +20,13 @@ program_desc "csv2psql #{Csv2Psql::VERSION} (Codename: #{Csv2Psql::CODENAME})"
20
20
  cmds = {
21
21
  h: {
22
22
  desc: 'Header row included',
23
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:header]
23
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['header']
24
24
  },
25
25
 
26
26
  d: {
27
27
  desc: 'Column delimiter',
28
28
  type: String,
29
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:delimiter]
29
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['delimiter']
30
30
  },
31
31
 
32
32
  l: {
@@ -38,13 +38,13 @@ cmds = {
38
38
  q: {
39
39
  desc: 'Quoting character',
40
40
  type: String,
41
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:quote]
41
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['quote']
42
42
  },
43
43
 
44
44
  s: {
45
45
  desc: 'Line separator',
46
46
  type: String,
47
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:separator]
47
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['separator']
48
48
  },
49
49
 
50
50
  'skip' => {
@@ -17,7 +17,7 @@ Csv2Psql::Cli.module_eval do
17
17
  res.files.each do |_fname, results|
18
18
  results[:columns].each do |_k, v|
19
19
  v.each do |d, det|
20
- v[d] = det.to_h
20
+ v[d] = det[:results]
21
21
  end
22
22
  end
23
23
  end
@@ -30,7 +30,7 @@ Csv2Psql::Cli.module_eval do
30
30
  header = ['column'] + res.analyzers.map { |a| a[:name] }
31
31
 
32
32
  rows = details[:columns].map do |k, v|
33
- [k] + v.keys.map { |name| v[name].count }
33
+ [k] + v.keys.map { |name| v[name][:results][:count] }
34
34
  end
35
35
 
36
36
  Terminal::Table.new title: file, headings: header, rows: rows
@@ -13,13 +13,13 @@ Csv2Psql::Cli.module_eval do
13
13
  cmds = {
14
14
  h: {
15
15
  desc: 'Header row included',
16
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:header]
16
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['header']
17
17
  },
18
18
 
19
19
  d: {
20
20
  desc: 'Column delimiter',
21
21
  type: String,
22
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:delimiter]
22
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['delimiter']
23
23
  },
24
24
 
25
25
  l: {
@@ -31,13 +31,13 @@ Csv2Psql::Cli.module_eval do
31
31
  q: {
32
32
  desc: 'Quoting character',
33
33
  type: String,
34
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:quote]
34
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['quote']
35
35
  },
36
36
 
37
37
  s: {
38
38
  desc: 'Line separator',
39
39
  type: String,
40
- default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:separator]
40
+ default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['separator']
41
41
  },
42
42
 
43
43
  'skip' => {
@@ -0,0 +1,24 @@
1
+ # encoding: UTF-8
2
+
3
+ require 'multi_json'
4
+
5
+ require_relative '../helpers/json_helper'
6
+
7
+ module Csv2Psql
8
+ # Configuration module
9
+ module Config
10
+ BASE_DIR = File.join(File.dirname(__FILE__), '..', '..', '..')
11
+ CONFIG_PATH = File.join(BASE_DIR, 'config', 'config.json')
12
+
13
+ class << self
14
+ def config(path = CONFIG_PATH)
15
+ @config ||= load_config(path)
16
+ @config
17
+ end
18
+
19
+ def load_config(path = CONFIG_PATH)
20
+ JsonHelper.load_file(path)
21
+ end
22
+ end
23
+ end
24
+ end
@@ -1,10 +1,5 @@
1
1
  # encoding: UTF-8
2
2
 
3
- require 'csv'
4
- require 'multi_json'
5
- require 'pathname'
6
- require 'pp'
7
-
8
3
  require_relative '../version'
9
4
  require_relative '../processor/processor'
10
5
 
@@ -4,7 +4,7 @@
4
4
  class String
5
5
  def camel_case
6
6
  return self if self !~ /_/ && self =~ /[A-Z]+.*/
7
- split('_').map { |e| e.capitalize }.join
7
+ split('_').map(&:capitalize).join
8
8
  end
9
9
 
10
10
  def camel_case_lower
@@ -0,0 +1,10 @@
1
+ # encoding: UTF-8
2
+
3
+ module Csv2Psql
4
+ # Frontend parsers
5
+ module Frontend
6
+ # Base class
7
+ class Base
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,19 @@
1
+ # encoding: UTF-8
2
+
3
+ require 'csv'
4
+
5
+ require_relative 'base'
6
+
7
+ module Csv2Psql
8
+ # Frontend parsers
9
+ module Frontend
10
+ # Csv frontend class
11
+ class Csv < Base
12
+ def open(path, open_opts = 'rt', csv_opts = {}, &_block)
13
+ CSV.open(path, open_opts, csv_opts) do |csv|
14
+ Proc.new.call(csv)
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,9 @@
1
+ # encoding: UTF-8
2
+
3
+ require_relative 'base'
4
+
5
+ module Csv2Psql
6
+ # Frontend parsers
7
+ module Frontend
8
+ end
9
+ end
@@ -1,7 +1,5 @@
1
1
  # encoding: UTF-8
2
2
 
3
- require 'csv'
4
- require 'multi_json'
5
3
  require 'pathname'
6
4
  require 'pp'
7
5
 
@@ -0,0 +1,14 @@
1
+ # encoding: UTF-8
2
+
3
+ require_relative '../config/config'
4
+
5
+ module Csv2Psql
6
+ # CSV Helper
7
+ class ConfigHelper
8
+ class << self
9
+ def config(path = Config::CONFIG_PATH)
10
+ Config.config(path)
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,31 @@
1
+ # encoding: UTF-8
2
+
3
+ require 'multi_json'
4
+
5
+ module Csv2Psql
6
+ # Json Helper
7
+ class JsonHelper
8
+ BASE_DIR = File.join(File.dirname(__FILE__), '..')
9
+
10
+ class << self
11
+ def load_file(path)
12
+ # Load input file
13
+ raw = IO.read(path)
14
+
15
+ # Try to parse json from loaded data
16
+ begin
17
+ return MultiJson.load(raw)
18
+ rescue Exception => e # rubocop:disable RescueException
19
+ log_exception(e)
20
+ raise e
21
+ end
22
+ nil
23
+ end
24
+
25
+ def log_exception(e)
26
+ puts 'Invalid json, see error.txt'
27
+ File.open('error.txt', 'wt') { |f| f.write(e.to_s) }
28
+ end
29
+ end
30
+ end
31
+ end
@@ -1,10 +1,5 @@
1
1
  # encoding: UTF-8
2
2
 
3
- require 'csv'
4
- require 'multi_json'
5
- require 'pathname'
6
- require 'pp'
7
-
8
3
  require_relative '../version'
9
4
  require_relative '../helpers/erb_helper'
10
5
 
@@ -1,12 +1,14 @@
1
1
  # encoding: UTF-8
2
2
 
3
3
  require 'csv'
4
- require 'multi_json'
5
4
  require 'pathname'
6
5
  require 'pp'
7
6
 
8
7
  require_relative '../analyzer/analyzer'
8
+ require_relative '../cache/cache'
9
+ require_relative '../frontend/csv'
9
10
  require_relative '../generator/generator'
11
+ require_relative '../helpers/config_helper'
10
12
  require_relative '../helpers/csv_helper'
11
13
  require_relative '../helpers/erb_helper'
12
14
  require_relative '../output/output'
@@ -17,18 +19,14 @@ module Csv2Psql
17
19
  class Processor
18
20
  attr_reader :analyzer, :generator, :output, :path
19
21
 
20
- DEFAULT_OPTIONS = {
21
- delimiter: ',',
22
- header: true,
23
- separator: :auto,
24
- transaction: false,
25
- quote: '"'
26
- }
22
+ DEFAULT_OPTIONS = ConfigHelper.config['processor']
27
23
 
28
24
  def initialize
29
25
  @output = Output.new
30
26
  @generator = Generator.new(@output)
31
- @analyzer = Analyzer.new
27
+ @cache = Cache.new
28
+ @analyzer = Analyzer.new(@cache)
29
+ @frontend = Frontend::Csv.new
32
30
  end
33
31
 
34
32
  def analyze(paths, opts = {})
@@ -73,23 +71,36 @@ module Csv2Psql
73
71
  end
74
72
 
75
73
  def merge_csv_options(opts = {})
76
- header = !opts[:header].nil? ? opts[:header] : DEFAULT_OPTIONS[:header]
77
- {
78
- col_sep: opts[:delimiter] || DEFAULT_OPTIONS[:delimiter],
74
+ header = !opts['header'].nil? ? opts['header'] : DEFAULT_OPTIONS['header']
75
+ res = {
79
76
  headers: header,
80
- row_sep: opts[:separator] || DEFAULT_OPTIONS[:separator],
81
- quote_char: opts[:quote] || DEFAULT_OPTIONS[:quote]
77
+ quote_char: opts['quote'] || DEFAULT_OPTIONS['quote']
82
78
  }
79
+ res[:col_sep] = opts['delimiter'] if opts['delimiter']
80
+ res[:row_sep] = opts['separator'] if opts['separator']
81
+ res
82
+ end
83
+
84
+ def process_file(path, csv, opts, &block)
85
+ lines = 0
86
+ limit = opts[:l]
87
+ skip = opts[:skip]
88
+ csv.each do |row|
89
+ lines += 1
90
+ next if skip > 0 && lines <= skip
91
+
92
+ with_row(path, row, opts, &block)
93
+
94
+ return if limit > 0 && lines >= limit
95
+ end
83
96
  end
84
97
 
85
98
  def with_path(path, opts = {}, &block)
86
99
  output.write 'BEGIN;' if opts[:transaction]
87
100
  csv_opts = merge_csv_options(opts)
88
101
  @first_row = true
89
- CSV.open(path, 'rt', csv_opts) do |csv|
90
- csv.each do |row|
91
- with_row(path, row, opts, &block)
92
- end
102
+ @frontend.open(path, 'rt', csv_opts) do |csv|
103
+ process_file(path, csv, opts, &block)
93
104
  end
94
105
  output.write 'COMMIT;' if opts[:transaction]
95
106
  end
@@ -101,9 +112,9 @@ module Csv2Psql
101
112
  end
102
113
  end
103
114
 
104
- def with_row(path, row, _opts = {}, &block)
115
+ def with_row(path, row, _opts = {}, &_block)
105
116
  args = { path: path, row: row }
106
- block.call(args) if block_given?
117
+ Proc.new.call(args)
107
118
  end
108
119
  end
109
120
  end
@@ -2,6 +2,6 @@
2
2
 
3
3
  # Csv2Psql module
4
4
  module Csv2Psql
5
- CODENAME = 'Lazy dog'
6
- VERSION = '0.0.10'
5
+ CODENAME = 'Famous rat'
6
+ VERSION = '0.0.11'
7
7
  end
@@ -7,11 +7,11 @@ module CliHelper
7
7
  # Execute block and capture its stdou
8
8
  # @param block Block to be executed with stdout redirected
9
9
  # @returns Captured output as string
10
- def capture_stdout(&block)
10
+ def capture_stdout(&_block)
11
11
  original_stdout = $stdout
12
12
  $stdout = fake = StringIO.new
13
13
  begin
14
- block.call if block_given?
14
+ Proc.new.call if block_given?
15
15
  ensure
16
16
  $stdout = original_stdout
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: csv2psql
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tomas Korcak
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-09-01 00:00:00.000000000 Z
11
+ date: 2014-09-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: gli
@@ -44,6 +44,26 @@ dependencies:
44
44
  - - "~>"
45
45
  - !ruby/object:Gem::Version
46
46
  version: 1.8.1
47
+ - !ruby/object:Gem::Dependency
48
+ name: lru
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '0.1'
54
+ - - ">="
55
+ - !ruby/object:Gem::Version
56
+ version: 0.1.0
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - "~>"
62
+ - !ruby/object:Gem::Version
63
+ version: '0.1'
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: 0.1.0
47
67
  - !ruby/object:Gem::Dependency
48
68
  name: multi_json
49
69
  requirement: !ruby/object:Gem::Requirement
@@ -244,29 +264,40 @@ files:
244
264
  - Rakefile
245
265
  - TODO.md
246
266
  - bin/csv2psql
267
+ - config/config.json
247
268
  - data/cia-data-all.csv
248
269
  - data/sample.csv
270
+ - data/sample_bool.csv
249
271
  - data/sample_semicolons.csv
250
272
  - lib/csv2psql.rb
251
273
  - lib/csv2psql/analyzer/analyzer.rb
274
+ - lib/csv2psql/analyzer/types/base_analyzer.rb
252
275
  - lib/csv2psql/analyzer/types/bigint.rb
276
+ - lib/csv2psql/analyzer/types/boolean.rb
253
277
  - lib/csv2psql/analyzer/types/character.rb
254
278
  - lib/csv2psql/analyzer/types/decimal.rb
255
279
  - lib/csv2psql/analyzer/types/null.rb
256
280
  - lib/csv2psql/analyzer/types/string.rb
257
281
  - lib/csv2psql/analyzer/types/uuid.rb
282
+ - lib/csv2psql/cache/cache.rb
258
283
  - lib/csv2psql/cli/app.rb
259
284
  - lib/csv2psql/cli/cli.rb
260
285
  - lib/csv2psql/cli/cmd/analyze_cmd.rb
261
286
  - lib/csv2psql/cli/cmd/convert_cmd.rb
262
287
  - lib/csv2psql/cli/cmd/version_cmd.rb
263
288
  - lib/csv2psql/cli/shared.rb
289
+ - lib/csv2psql/config/config.rb
264
290
  - lib/csv2psql/convert/convert.rb
265
- - lib/csv2psql/dialects/psql.rb
266
291
  - lib/csv2psql/extensions/string.rb
292
+ - lib/csv2psql/frontend/base.rb
293
+ - lib/csv2psql/frontend/csv.rb
294
+ - lib/csv2psql/frontend/frontend.rb
295
+ - lib/csv2psql/generator/dialects/psql.rb
267
296
  - lib/csv2psql/generator/generator.rb
297
+ - lib/csv2psql/helpers/config_helper.rb
268
298
  - lib/csv2psql/helpers/csv_helper.rb
269
299
  - lib/csv2psql/helpers/erb_helper.rb
300
+ - lib/csv2psql/helpers/json_helper.rb
270
301
  - lib/csv2psql/lib.rb
271
302
  - lib/csv2psql/output/output.rb
272
303
  - lib/csv2psql/processor/processor.rb