csv2psql 0.0.12 → 0.0.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/bin/csv2psql +0 -4
- data/data/census_SFOH_2010.csv +981 -981
- data/data/sfpd_incident_2010_1k.csv +1001 -0
- data/lib/csv2psql/cli/app.rb +1 -3
- data/lib/csv2psql/cli/cli.rb +1 -29
- data/lib/csv2psql/cli/cmd/analyze_cmd.rb +42 -48
- data/lib/csv2psql/cli/cmd/convert_cmd.rb +37 -44
- data/lib/csv2psql/cli/cmd/schema_cmd.rb +52 -58
- data/lib/csv2psql/cli/cmd/version_cmd.rb +4 -12
- data/lib/csv2psql/cli/shared.rb +44 -46
- data/lib/csv2psql/version.rb +2 -2
- data/lib/csv2psql.rb +1 -1
- data/templates/schema.sql.erb +1 -1
- metadata +4 -3
data/lib/csv2psql/cli/app.rb
CHANGED
@@ -20,8 +20,6 @@ module Csv2Psql
|
|
20
20
|
module Cli
|
21
21
|
# CLI Application
|
22
22
|
class App
|
23
|
-
extend Csv2Psql::Cli::Shared
|
24
|
-
|
25
23
|
cmds = File.absolute_path(File.join(File.dirname(__FILE__), 'cmd'))
|
26
24
|
Dir.glob(cmds + '/*.rb').each do |file|
|
27
25
|
require file
|
@@ -34,4 +32,4 @@ module Csv2Psql
|
|
34
32
|
end
|
35
33
|
end
|
36
34
|
|
37
|
-
launch
|
35
|
+
launch
|
data/lib/csv2psql/cli/cli.rb
CHANGED
@@ -1,31 +1,3 @@
|
|
1
1
|
# encoding: UTF-8
|
2
2
|
|
3
|
-
|
4
|
-
require 'pp'
|
5
|
-
|
6
|
-
# Define Csv2Psql::Cli as GLI Wrapper
|
7
|
-
module Csv2Psql
|
8
|
-
# CLI/GLI Wrapper
|
9
|
-
module Cli
|
10
|
-
include GLI::App
|
11
|
-
|
12
|
-
# Require shared part of GLI::App - flags, meta, etc
|
13
|
-
require_relative 'shared.rb'
|
14
|
-
|
15
|
-
# Require Hooks
|
16
|
-
# require_relative 'hooks.rb'
|
17
|
-
|
18
|
-
base = File.join(File.dirname(__FILE__), 'cmd')
|
19
|
-
Dir[base + '/**/*.rb'].each do |f|
|
20
|
-
require f
|
21
|
-
end
|
22
|
-
|
23
|
-
# GLI::App.commands_from(File.join(File.dirname(__FILE__), 'cmd'))
|
24
|
-
|
25
|
-
def self.main(args = ARGV)
|
26
|
-
run(args)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
Csv2Psql::Cli.main(ARGV) if __FILE__ == $PROGRAM_NAME
|
3
|
+
require_relative 'app'
|
@@ -1,76 +1,70 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
3
|
require 'json'
|
5
4
|
require 'pp'
|
6
5
|
require 'terminal-table'
|
7
6
|
|
8
|
-
include GLI::App
|
9
|
-
|
10
|
-
require_relative '../shared'
|
11
7
|
require_relative '../../convert/convert'
|
12
8
|
require_relative '../../processor/processor'
|
13
9
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
v
|
20
|
-
v[d] = det[:results]
|
21
|
-
end
|
10
|
+
formats = {
|
11
|
+
'json' => lambda do |res|
|
12
|
+
res.files.each do |_fname, results|
|
13
|
+
results[:columns].each do |_k, v|
|
14
|
+
v.each do |d, det|
|
15
|
+
v[d] = det[:results]
|
22
16
|
end
|
23
17
|
end
|
18
|
+
end
|
24
19
|
|
25
|
-
|
26
|
-
|
20
|
+
JSON.pretty_generate(res.files)
|
21
|
+
end,
|
27
22
|
|
28
|
-
|
29
|
-
|
30
|
-
|
23
|
+
'table' => lambda do |res|
|
24
|
+
res.files.map do |file, details|
|
25
|
+
header = ['column'] + res.analyzers.map { |a| a[:name] }
|
31
26
|
|
32
|
-
|
33
|
-
|
34
|
-
end
|
35
|
-
|
36
|
-
Terminal::Table.new title: file, headings: header, rows: rows
|
27
|
+
rows = details[:columns].map do |k, v|
|
28
|
+
[k] + v.keys.map { |name| v[name][:results][:count] }
|
37
29
|
end
|
30
|
+
|
31
|
+
Terminal::Table.new title: file, headings: header, rows: rows
|
38
32
|
end
|
39
|
-
|
33
|
+
end
|
34
|
+
}
|
40
35
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
}
|
36
|
+
cmds = {
|
37
|
+
f: {
|
38
|
+
desc: 'Output format',
|
39
|
+
type: String,
|
40
|
+
default_value: formats.keys.first
|
47
41
|
}
|
42
|
+
}
|
48
43
|
|
49
|
-
|
50
|
-
|
51
|
-
|
44
|
+
desc 'Analyze csv file'
|
45
|
+
command :analyze do |c|
|
46
|
+
c.flag [:f, :format], cmds[:f]
|
52
47
|
|
53
|
-
|
54
|
-
|
48
|
+
c.action do |global_options, options, args|
|
49
|
+
fail ArgumentError, 'No file to analyze specified' if args.empty?
|
55
50
|
|
56
|
-
|
51
|
+
opts = {}.merge(global_options).merge(options)
|
57
52
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
53
|
+
formater = formats[opts[:format]]
|
54
|
+
if formater.nil?
|
55
|
+
fmters = formats.keys.join(', ')
|
56
|
+
fail ArgumentError, "Wrong formatter specified, can be: #{fmters}"
|
57
|
+
end
|
63
58
|
|
64
|
-
|
59
|
+
res = Csv2Psql::Convert.analyze(args, opts)
|
65
60
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
end
|
71
|
-
else
|
72
|
-
puts output
|
61
|
+
output = formater.call(res)
|
62
|
+
if output.is_a?(Array)
|
63
|
+
output.each do |o|
|
64
|
+
puts o
|
73
65
|
end
|
66
|
+
else
|
67
|
+
puts output
|
74
68
|
end
|
75
69
|
end
|
76
70
|
end
|
@@ -1,55 +1,48 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
|
-
|
5
|
-
include GLI::App
|
6
|
-
|
7
|
-
require_relative '../shared'
|
8
3
|
require_relative '../../convert/convert'
|
9
4
|
require_relative '../../processor/processor'
|
10
5
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
'
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
'
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
'
|
35
|
-
|
36
|
-
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['truncate-table']
|
37
|
-
}
|
6
|
+
cmds = {
|
7
|
+
t: {
|
8
|
+
desc: 'Table to insert to',
|
9
|
+
type: String,
|
10
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:table]
|
11
|
+
},
|
12
|
+
|
13
|
+
transaction: {
|
14
|
+
desc: 'Import in transaction block',
|
15
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS[:transaction]
|
16
|
+
},
|
17
|
+
|
18
|
+
'create-table' => {
|
19
|
+
desc: 'Crate SQL Table before inserts',
|
20
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['create-table']
|
21
|
+
},
|
22
|
+
|
23
|
+
'drop-table' => {
|
24
|
+
desc: 'Drop SQL Table before inserts',
|
25
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['drop-table']
|
26
|
+
},
|
27
|
+
|
28
|
+
'truncate-table' => {
|
29
|
+
desc: 'Truncate SQL Table before inserts',
|
30
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['truncate-table']
|
38
31
|
}
|
32
|
+
}
|
39
33
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
34
|
+
desc 'Convert csv file'
|
35
|
+
command :convert do |c|
|
36
|
+
c.flag [:t, :table], cmds[:t]
|
37
|
+
c.switch [:transaction], cmds[:transaction]
|
38
|
+
c.switch ['create-table'], cmds['create-table']
|
39
|
+
c.switch ['drop-table'], cmds['drop-table']
|
40
|
+
c.switch ['truncate-table'], cmds['truncate-table']
|
47
41
|
|
48
|
-
|
49
|
-
|
42
|
+
c.action do |global_options, options, args|
|
43
|
+
fail ArgumentError, 'No file to convert specified' if args.empty?
|
50
44
|
|
51
|
-
|
52
|
-
|
53
|
-
end
|
45
|
+
opts = {}.merge(global_options).merge(options)
|
46
|
+
Csv2Psql::Convert.convert(args, opts)
|
54
47
|
end
|
55
48
|
end
|
@@ -1,85 +1,79 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
3
|
require 'json'
|
5
4
|
require 'pp'
|
6
5
|
require 'terminal-table'
|
7
6
|
|
8
|
-
include GLI::App
|
9
|
-
|
10
|
-
require_relative '../shared'
|
11
7
|
require_relative '../../convert/convert'
|
12
8
|
require_relative '../../helpers/erb_helper'
|
13
9
|
require_relative '../../processor/processor'
|
14
10
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
11
|
+
BASE_DIR = File.join(File.dirname(__FILE__), '..', '..', '..', '..')
|
12
|
+
TEMPLATE_DIR = File.join(BASE_DIR, 'templates')
|
13
|
+
SCHEMA_TEMPLATE = File.join(TEMPLATE_DIR, 'schema.sql.erb')
|
14
|
+
|
15
|
+
formats = {
|
16
|
+
'json' => lambda do |res|
|
17
|
+
JSON.pretty_generate(res)
|
18
|
+
end,
|
19
|
+
|
20
|
+
'sql' => lambda do |data|
|
21
|
+
res = ''
|
22
|
+
data.each do |_k, v|
|
23
|
+
v[:table] = 'my_table'
|
24
|
+
ctx = v
|
25
|
+
erb = Csv2Psql::ErbHelper.new
|
26
|
+
res += "\n" unless res.empty?
|
27
|
+
res += erb.process(SCHEMA_TEMPLATE, ctx)
|
28
|
+
end
|
29
|
+
res
|
30
|
+
end,
|
19
31
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
end,
|
32
|
+
'table' => lambda do |res|
|
33
|
+
res.map do |file, data|
|
34
|
+
header = %w(column type null)
|
24
35
|
|
25
|
-
|
26
|
-
|
27
|
-
data.each do |_k, v|
|
28
|
-
v[:table] = 'my_table'
|
29
|
-
ctx = v
|
30
|
-
erb = Csv2Psql::ErbHelper.new
|
31
|
-
res += "\n" unless res.empty?
|
32
|
-
res += erb.process(SCHEMA_TEMPLATE, ctx)
|
36
|
+
rows = data[:columns].map do |k, v|
|
37
|
+
[k, v[:type], v[:null]]
|
33
38
|
end
|
34
|
-
res
|
35
|
-
end,
|
36
|
-
|
37
|
-
'table' => lambda do |res|
|
38
|
-
res.map do |file, data|
|
39
|
-
header = %w(column type null)
|
40
39
|
|
41
|
-
|
42
|
-
[k, v[:type], v[:null]]
|
43
|
-
end
|
44
|
-
|
45
|
-
Terminal::Table.new title: file, headings: header, rows: rows
|
46
|
-
end
|
40
|
+
Terminal::Table.new title: file, headings: header, rows: rows
|
47
41
|
end
|
48
|
-
|
42
|
+
end
|
43
|
+
}
|
49
44
|
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
}
|
45
|
+
cmds = {
|
46
|
+
f: {
|
47
|
+
desc: 'Output format',
|
48
|
+
type: String,
|
49
|
+
default_value: formats.keys.first
|
56
50
|
}
|
51
|
+
}
|
57
52
|
|
58
|
-
|
59
|
-
|
60
|
-
|
53
|
+
desc 'Generate schema for file'
|
54
|
+
command :schema do |c|
|
55
|
+
c.flag [:f, :format], cmds[:f]
|
61
56
|
|
62
|
-
|
63
|
-
|
57
|
+
c.action do |global_options, options, args|
|
58
|
+
fail ArgumentError, 'No file to analyze specified' if args.empty?
|
64
59
|
|
65
|
-
|
60
|
+
opts = {}.merge(global_options).merge(options)
|
66
61
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
62
|
+
formater = formats[opts[:format]]
|
63
|
+
if formater.nil?
|
64
|
+
fmters = formats.keys.join(', ')
|
65
|
+
fail ArgumentError, "Wrong formatter specified, can be: #{fmters}"
|
66
|
+
end
|
72
67
|
|
73
|
-
|
68
|
+
res = Csv2Psql::Convert.generate_schema(args, opts)
|
74
69
|
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
else
|
81
|
-
puts output
|
70
|
+
output = formater.call(res)
|
71
|
+
if output.is_a?(Array)
|
72
|
+
output.each do |o|
|
73
|
+
puts o
|
82
74
|
end
|
75
|
+
else
|
76
|
+
puts output
|
83
77
|
end
|
84
78
|
end
|
85
79
|
end
|
@@ -1,18 +1,10 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
|
-
require 'gli'
|
4
|
-
|
5
|
-
include GLI::App
|
6
|
-
|
7
3
|
require_relative '../../version'
|
8
4
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
command :version do |c|
|
14
|
-
c.action do |_global_options, _options, _args|
|
15
|
-
pp Csv2Psql::VERSION
|
16
|
-
end
|
5
|
+
desc 'Print version info'
|
6
|
+
command :version do |c|
|
7
|
+
c.action do |_global_options, _options, _args|
|
8
|
+
pp Csv2Psql::VERSION
|
17
9
|
end
|
18
10
|
end
|
data/lib/csv2psql/cli/shared.rb
CHANGED
@@ -7,50 +7,48 @@ require_relative '../processor/processor'
|
|
7
7
|
|
8
8
|
include GLI::App
|
9
9
|
|
10
|
-
Csv2Psql::
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
'skip'
|
44
|
-
|
45
|
-
|
46
|
-
default_value: -1
|
47
|
-
}
|
10
|
+
program_desc "csv2psql #{Csv2Psql::VERSION} (Codename: #{Csv2Psql::CODENAME})"
|
11
|
+
|
12
|
+
cmds = {
|
13
|
+
h: {
|
14
|
+
desc: 'Header row included',
|
15
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['header']
|
16
|
+
},
|
17
|
+
|
18
|
+
d: {
|
19
|
+
desc: 'Column delimiter',
|
20
|
+
type: String,
|
21
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['delimiter']
|
22
|
+
},
|
23
|
+
|
24
|
+
l: {
|
25
|
+
desc: 'How many rows process',
|
26
|
+
type: Integer,
|
27
|
+
default_value: -1
|
28
|
+
},
|
29
|
+
|
30
|
+
q: {
|
31
|
+
desc: 'Quoting character',
|
32
|
+
type: String,
|
33
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['quote']
|
34
|
+
},
|
35
|
+
|
36
|
+
s: {
|
37
|
+
desc: 'Line separator',
|
38
|
+
type: String,
|
39
|
+
default_value: Csv2Psql::Processor::DEFAULT_OPTIONS['separator']
|
40
|
+
},
|
41
|
+
|
42
|
+
'skip' => {
|
43
|
+
desc: 'How many rows skip',
|
44
|
+
type: Integer,
|
45
|
+
default_value: -1
|
48
46
|
}
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
47
|
+
}
|
48
|
+
|
49
|
+
switch [:h, :header], cmds[:h]
|
50
|
+
flag [:d, :delimiter], cmds[:d]
|
51
|
+
flag [:l, :limit], cmds[:l]
|
52
|
+
flag [:q, :quote], cmds[:q]
|
53
|
+
flag [:s, :separator], cmds[:s]
|
54
|
+
flag [:skip], cmds['skip']
|
data/lib/csv2psql/version.rb
CHANGED
data/lib/csv2psql.rb
CHANGED
data/templates/schema.sql.erb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
CREATE TABLE <%= ctx[:table] %>
|
2
2
|
(
|
3
3
|
<% ctx[:columns].each_with_index do |item, index| %>
|
4
|
-
"<%= Generator.sanitize_header(item[0]) %>" <%= item[1][:type]
|
4
|
+
"<%= Generator.sanitize_header(item[0]) %>" <%= item[1][:type].upcase %><% if !item[1][:null] %> NOT NULL<% end %><%= ', ' if index < ctx[:columns].length - 1%>
|
5
5
|
|
6
6
|
<% end %>
|
7
7
|
)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: csv2psql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tomas Korcak
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-11-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gli
|
@@ -270,6 +270,7 @@ files:
|
|
270
270
|
- data/sample.csv
|
271
271
|
- data/sample_bool.csv
|
272
272
|
- data/sample_semicolons.csv
|
273
|
+
- data/sfpd_incident_2010_1k.csv
|
273
274
|
- lib/csv2psql.rb
|
274
275
|
- lib/csv2psql/analyzer/analyzer.rb
|
275
276
|
- lib/csv2psql/analyzer/types/base_analyzer.rb
|
@@ -336,7 +337,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
336
337
|
version: '0'
|
337
338
|
requirements: []
|
338
339
|
rubyforge_project:
|
339
|
-
rubygems_version: 2.
|
340
|
+
rubygems_version: 2.4.6
|
340
341
|
signing_key:
|
341
342
|
specification_version: 4
|
342
343
|
summary: Tool for converting CSV into SQL statements
|