csvsql 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/.rspec +3 -0
- data/.rubocop.yml +137 -0
- data/.travis.yml +5 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +57 -0
- data/LICENSE.txt +21 -0
- data/README.md +112 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/csvsql.gemspec +30 -0
- data/exe/csvsql +34 -0
- data/lib/csvsql/db.rb +147 -0
- data/lib/csvsql/version.rb +5 -0
- data/lib/csvsql.rb +20 -0
- metadata +130 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 77f9bf18a9ec8773b4015b456cd4dc789bebd36d
|
4
|
+
data.tar.gz: 6c18bed0e815307ffa18cd8b9ef80a774abecaa9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 567347889397e591db8a04dc8dd9b8154e4915eb85b88575f62767a7e78bc578e2282d8250cd27f5947cc7462e3bd37f6869c455ec76ada203be4c8e03b1da7b
|
7
|
+
data.tar.gz: 4c345a17fb5b380d6039fbb1bfad0609596a1703c8dca07b65cd917f5bfa0b77d49e7d3b41181d6f6cf7195b8ebc8f5d869099aebfc0d2b8dc190de64e303656
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,137 @@
|
|
1
|
+
# https://github.com/rails/rails/blob/master/.rubocop.yml
|
2
|
+
#
|
3
|
+
AllCops:
|
4
|
+
TargetRubyVersion: 2.4
|
5
|
+
# RuboCop has a bunch of cops enabled by default. This setting tells RuboCop
|
6
|
+
# to ignore them, so only the ones explicitly set in this file are enabled.
|
7
|
+
DisabledByDefault: true
|
8
|
+
|
9
|
+
# Prefer &&/|| over and/or.
|
10
|
+
Style/AndOr:
|
11
|
+
Enabled: true
|
12
|
+
|
13
|
+
# Do not use braces for hash literals when they are the last argument of a
|
14
|
+
# method call.
|
15
|
+
Style/BracesAroundHashParameters:
|
16
|
+
Enabled: true
|
17
|
+
|
18
|
+
# Align `when` with `case`.
|
19
|
+
Layout/CaseIndentation:
|
20
|
+
Enabled: true
|
21
|
+
EnforcedStyle: end
|
22
|
+
IndentOneStep: false
|
23
|
+
|
24
|
+
# Align comments with method definitions.
|
25
|
+
Layout/CommentIndentation:
|
26
|
+
Enabled: true
|
27
|
+
|
28
|
+
Layout/EmptyLineAfterMagicComment:
|
29
|
+
Enabled: true
|
30
|
+
|
31
|
+
# No extra empty lines.
|
32
|
+
Layout/EmptyLines:
|
33
|
+
Enabled: true
|
34
|
+
|
35
|
+
# In a regular class definition, no empty lines around the body.
|
36
|
+
Layout/EmptyLinesAroundClassBody:
|
37
|
+
Enabled: true
|
38
|
+
|
39
|
+
# In a regular method definition, no empty lines around the body.
|
40
|
+
Layout/EmptyLinesAroundMethodBody:
|
41
|
+
Enabled: true
|
42
|
+
|
43
|
+
# In a regular module definition, no empty lines around the body.
|
44
|
+
Layout/EmptyLinesAroundModuleBody:
|
45
|
+
Enabled: true
|
46
|
+
|
47
|
+
# Use Ruby >= 1.9 syntax for hashes. Prefer { a: :b } over { :a => :b }.
|
48
|
+
Style/HashSyntax:
|
49
|
+
Enabled: true
|
50
|
+
|
51
|
+
# Method definitions after `private` or `protected` isolated calls need one
|
52
|
+
# extra level of indentation.
|
53
|
+
Layout/IndentationConsistency:
|
54
|
+
Enabled: true
|
55
|
+
# EnforcedStyle: rails
|
56
|
+
|
57
|
+
# Two spaces, no tabs (for indentation).
|
58
|
+
Layout/IndentationWidth:
|
59
|
+
Enabled: true
|
60
|
+
|
61
|
+
Layout/SpaceAfterColon:
|
62
|
+
Enabled: true
|
63
|
+
|
64
|
+
Layout/SpaceAfterComma:
|
65
|
+
Enabled: true
|
66
|
+
|
67
|
+
Layout/SpaceAroundEqualsInParameterDefault:
|
68
|
+
Enabled: true
|
69
|
+
|
70
|
+
Layout/SpaceAroundKeyword:
|
71
|
+
Enabled: true
|
72
|
+
|
73
|
+
Layout/SpaceAroundOperators:
|
74
|
+
Enabled: true
|
75
|
+
|
76
|
+
Layout/SpaceBeforeFirstArg:
|
77
|
+
Enabled: true
|
78
|
+
|
79
|
+
# Defining a method with parameters needs parentheses.
|
80
|
+
Style/MethodDefParentheses:
|
81
|
+
Enabled: true
|
82
|
+
|
83
|
+
# str = "abcda"
|
84
|
+
# str.gsub!("c", "b") # will raise a error, we cannot change string literal
|
85
|
+
Style/FrozenStringLiteralComment:
|
86
|
+
Enabled: true
|
87
|
+
EnforcedStyle: always
|
88
|
+
Include:
|
89
|
+
- 'app/**/*'
|
90
|
+
- 'lib/**/*'
|
91
|
+
|
92
|
+
# Use `foo {}` not `foo{}`.
|
93
|
+
Layout/SpaceBeforeBlockBraces:
|
94
|
+
Enabled: true
|
95
|
+
|
96
|
+
# Use `foo { bar }` not `foo {bar}`.
|
97
|
+
Layout/SpaceInsideBlockBraces:
|
98
|
+
Enabled: true
|
99
|
+
|
100
|
+
# Use `{ a: 1 }` not `{a:1}`.
|
101
|
+
Layout/SpaceInsideHashLiteralBraces:
|
102
|
+
Enabled: true
|
103
|
+
|
104
|
+
Layout/SpaceInsideParens:
|
105
|
+
Enabled: true
|
106
|
+
|
107
|
+
# Check quotes usage according to lint rule below.
|
108
|
+
# Style/StringLiterals:
|
109
|
+
# Enabled: true
|
110
|
+
# EnforcedStyle: double_quotes
|
111
|
+
|
112
|
+
# Detect hard tabs, no hard tabs.
|
113
|
+
Layout/Tab:
|
114
|
+
Enabled: true
|
115
|
+
|
116
|
+
# Blank lines should not have any spaces.
|
117
|
+
Layout/TrailingBlankLines:
|
118
|
+
Enabled: true
|
119
|
+
|
120
|
+
# No trailing whitespace.
|
121
|
+
Layout/TrailingWhitespace:
|
122
|
+
Enabled: true
|
123
|
+
|
124
|
+
# Use quotes for string literals when they are enough.
|
125
|
+
# Style/UnneededPercentQ:
|
126
|
+
# Enabled: true
|
127
|
+
|
128
|
+
# Align `end` with the matching keyword or starting expression except for
|
129
|
+
# assignments, where it should be aligned with the LHS.
|
130
|
+
Lint/EndAlignment:
|
131
|
+
Enabled: true
|
132
|
+
EnforcedStyleAlignWith: variable
|
133
|
+
|
134
|
+
# Use my_method(my_arg) not my_method( my_arg ) or my_method my_arg.
|
135
|
+
Lint/RequireParentheses:
|
136
|
+
Enabled: true
|
137
|
+
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
csvsql (0.1.0)
|
5
|
+
sqlite3 (~> 1.3.13)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
ast (2.4.0)
|
11
|
+
coderay (1.1.2)
|
12
|
+
diff-lcs (1.3)
|
13
|
+
method_source (0.9.0)
|
14
|
+
parallel (1.12.1)
|
15
|
+
parser (2.5.1.2)
|
16
|
+
ast (~> 2.4.0)
|
17
|
+
powerpack (0.1.2)
|
18
|
+
pry (0.11.3)
|
19
|
+
coderay (~> 1.1.0)
|
20
|
+
method_source (~> 0.9.0)
|
21
|
+
rainbow (3.0.0)
|
22
|
+
rspec (3.7.0)
|
23
|
+
rspec-core (~> 3.7.0)
|
24
|
+
rspec-expectations (~> 3.7.0)
|
25
|
+
rspec-mocks (~> 3.7.0)
|
26
|
+
rspec-core (3.7.1)
|
27
|
+
rspec-support (~> 3.7.0)
|
28
|
+
rspec-expectations (3.7.0)
|
29
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
30
|
+
rspec-support (~> 3.7.0)
|
31
|
+
rspec-mocks (3.7.0)
|
32
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
+
rspec-support (~> 3.7.0)
|
34
|
+
rspec-support (3.7.1)
|
35
|
+
rubocop (0.52.1)
|
36
|
+
parallel (~> 1.10)
|
37
|
+
parser (>= 2.4.0.2, < 3.0)
|
38
|
+
powerpack (~> 0.1)
|
39
|
+
rainbow (>= 2.2.2, < 4.0)
|
40
|
+
ruby-progressbar (~> 1.7)
|
41
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
42
|
+
ruby-progressbar (1.9.0)
|
43
|
+
sqlite3 (1.3.13)
|
44
|
+
unicode-display_width (1.4.0)
|
45
|
+
|
46
|
+
PLATFORMS
|
47
|
+
ruby
|
48
|
+
|
49
|
+
DEPENDENCIES
|
50
|
+
bundler (~> 1.16.a)
|
51
|
+
csvsql!
|
52
|
+
pry
|
53
|
+
rspec (~> 3.0)
|
54
|
+
rubocop (~> 0.52.1)
|
55
|
+
|
56
|
+
BUNDLED WITH
|
57
|
+
1.16.0.pre.2
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2018 jiangzhi.xie
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
Csvsql
|
2
|
+
======
|
3
|
+
|
4
|
+
Use SQL to query your CSV file, and return a new CSV.
|
5
|
+
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'csvsql'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install csvsql
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
### Example CSV `mydata.csv`
|
26
|
+
|
27
|
+
```csv
|
28
|
+
name,total:int,created_at:datetime
|
29
|
+
a,12,2018-1-1 11:22
|
30
|
+
b,21,2018-3-1 10:20
|
31
|
+
c,39,2018-1-19 20:10
|
32
|
+
```
|
33
|
+
|
34
|
+
The first line is title for each columns. We use `:` split the title name and column type. the column type is `varchar(255)` if you did give a type.
|
35
|
+
|
36
|
+
There are same types: `int`, `integer` `float`, `double`, `date`, `datetime`, `varchar(255)`.
|
37
|
+
|
38
|
+
All sqlite3 type is supported, but maybe some type cannot convert to a sqlite3 value type.
|
39
|
+
|
40
|
+
### Query from file & Output
|
41
|
+
|
42
|
+
```
|
43
|
+
# csvsql [options] SQL
|
44
|
+
csvsql -h # Get help
|
45
|
+
```
|
46
|
+
|
47
|
+
```
|
48
|
+
$ csvsql -i mydata.csv "select * from csv where created_at > '2018-1-1' and total > 20" > /tmp/result.csv
|
49
|
+
# name:varchar(255),total:int,created_at:datetime
|
50
|
+
# b,21,2018-3-1 10:20:00
|
51
|
+
# c,39,2018-1-19 20:10:00
|
52
|
+
|
53
|
+
csvsql -i mydata.csv "select count(*) as total_record from csv where created_at > '2018-1-1'" > /tmp/result.csv
|
54
|
+
# total_record:integer
|
55
|
+
# 2
|
56
|
+
|
57
|
+
csvsql -i mydata.csv "select name, total from csv where total < 30" > /tmp/result.csv
|
58
|
+
# name:varchar(255),total:integer
|
59
|
+
# a,12
|
60
|
+
# b,21
|
61
|
+
```
|
62
|
+
|
63
|
+
### Query from stdin
|
64
|
+
|
65
|
+
If not give a csv by `-i`, we will get the input from stdin.
|
66
|
+
|
67
|
+
```
|
68
|
+
csvsql -i mydata.csv "select name, total from csv where total < 30" | csvsql "select count(*) as count from csv"
|
69
|
+
# count:integer
|
70
|
+
# 2
|
71
|
+
```
|
72
|
+
|
73
|
+
### Cache CSV data
|
74
|
+
|
75
|
+
It will save the CSV data to a tempfile. we use `~/.csvsql_cache` folder to save the cache
|
76
|
+
|
77
|
+
```
|
78
|
+
csvsql -i large.csv -t "select count(*) from csv"
|
79
|
+
|
80
|
+
# the second, it will be fast.
|
81
|
+
csvsql -i large.csv -t "select count(*) from csv"
|
82
|
+
```
|
83
|
+
|
84
|
+
### Clear Cache
|
85
|
+
|
86
|
+
This command will remove all data in the `~/.csvsql_cache`
|
87
|
+
|
88
|
+
```
|
89
|
+
csvsql --clear
|
90
|
+
```
|
91
|
+
|
92
|
+
|
93
|
+
## Performance TODO
|
94
|
+
|
95
|
+
* 10,000 lines
|
96
|
+
* 10,000 lines with cache
|
97
|
+
* 1,000,000 lines
|
98
|
+
* 1,000,000 lines with cache
|
99
|
+
|
100
|
+
|
101
|
+
## Development
|
102
|
+
|
103
|
+
* Make sure your code has some testing.
|
104
|
+
* Run `rubocop -a` before commit your code.
|
105
|
+
|
106
|
+
## Contributing
|
107
|
+
|
108
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/csvsql.
|
109
|
+
|
110
|
+
## License
|
111
|
+
|
112
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "csvsql"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/csvsql.gemspec
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "csvsql/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "csvsql"
|
8
|
+
spec.version = Csvsql::VERSION
|
9
|
+
spec.authors = ["jiangzhi.xie"]
|
10
|
+
spec.email = ["xiejiangzhi@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Process csv with SQL.}
|
13
|
+
spec.description = %q{Process csv with SQL.}
|
14
|
+
spec.homepage = "https://github.com/xiejiangzhi/csvsql"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
+
f.match(%r{^(test|spec|features)/})
|
19
|
+
end
|
20
|
+
spec.bindir = "exe"
|
21
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
|
+
spec.require_paths = ["lib"]
|
23
|
+
|
24
|
+
spec.add_dependency "sqlite3", "~> 1.3.13"
|
25
|
+
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.16.a"
|
27
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
28
|
+
spec.add_development_dependency "pry"
|
29
|
+
spec.add_development_dependency "rubocop", '~> 0.52.1'
|
30
|
+
end
|
data/exe/csvsql
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
lib = File.expand_path("../../lib", __FILE__)
|
6
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
7
|
+
require 'csvsql'
|
8
|
+
|
9
|
+
options = {}
|
10
|
+
OptionParser.new do |opts|
|
11
|
+
opts.banner = "Usage: csvsql [options] SQL"
|
12
|
+
|
13
|
+
opts.on('-i', '--input path', "CSV file path, optional. read from stdin if no give") do |path|
|
14
|
+
options[:csv_path] = path
|
15
|
+
end
|
16
|
+
|
17
|
+
opts.on('-c', '--use-cache', "Cache data in ~/.csvsql_cache. it will still reload if file was changed") do
|
18
|
+
options[:use_cache] = true
|
19
|
+
end
|
20
|
+
|
21
|
+
opts.on('--clear-cache', "Clear all cache data") do
|
22
|
+
options[:clear_cache] = true
|
23
|
+
end
|
24
|
+
end.parse!
|
25
|
+
|
26
|
+
if options[:clear_cache]
|
27
|
+
Csvsql::Db.clear_cache!
|
28
|
+
puts "Completed clear cache."
|
29
|
+
exit
|
30
|
+
end
|
31
|
+
|
32
|
+
csv_data = options[:csv_path] || StringIO.new($stdin.read)
|
33
|
+
|
34
|
+
puts Csvsql.execute(ARGV[0], csv_data, use_cache: options[:use_cache])
|
data/lib/csvsql/db.rb
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'digest'
|
4
|
+
|
5
|
+
module Csvsql
|
6
|
+
class Db
|
7
|
+
BATCH_LINES = 10000
|
8
|
+
CACHE_DIR = File.join(Dir.home, '.csvsql_cache')
|
9
|
+
|
10
|
+
FileUtils.mkdir_p(CACHE_DIR) unless Dir.exists?(CACHE_DIR)
|
11
|
+
|
12
|
+
attr_reader :use_cache, :csv_path, :csv_io, :db
|
13
|
+
|
14
|
+
def self.clear_cache!
|
15
|
+
require 'fileutils'
|
16
|
+
FileUtils.rm_f(Dir.glob(File.join(CACHE_DIR, '*')))
|
17
|
+
end
|
18
|
+
|
19
|
+
def initialize(use_cache: false)
|
20
|
+
@db = nil
|
21
|
+
@csv_path = nil
|
22
|
+
@use_cache = use_cache
|
23
|
+
end
|
24
|
+
|
25
|
+
# action:
|
26
|
+
# raise: default
|
27
|
+
# exit
|
28
|
+
def sql_error_action=(action)
|
29
|
+
@sql_error_action = action.to_sym
|
30
|
+
end
|
31
|
+
|
32
|
+
def execute(sql)
|
33
|
+
db.execute(sql)
|
34
|
+
rescue SQLite3::SQLException => e
|
35
|
+
process_sql_error(sql, e)
|
36
|
+
end
|
37
|
+
|
38
|
+
def prepare(sql)
|
39
|
+
db.prepare(sql)
|
40
|
+
rescue SQLite3::SQLException => e
|
41
|
+
process_sql_error(sql, e)
|
42
|
+
end
|
43
|
+
|
44
|
+
def import(csv_data_or_path)
|
45
|
+
case csv_data_or_path
|
46
|
+
when StringIO, IO
|
47
|
+
@csv_io = csv_data_or_path
|
48
|
+
else
|
49
|
+
@csv_path = csv_data_or_path
|
50
|
+
end
|
51
|
+
@db = SQLite3::Database.new(get_db_path(@csv_path))
|
52
|
+
|
53
|
+
tables = db.execute("SELECT name FROM sqlite_master WHERE type='table';").first
|
54
|
+
unless tables && tables.include?('csv')
|
55
|
+
init_db_by_csv(@csv_io ? CSV.new(@csv_io) : CSV.open(@csv_path))
|
56
|
+
end
|
57
|
+
true
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def parser_header(csv_header)
|
63
|
+
csv_header.map do |col, r|
|
64
|
+
name, type = col.strip.split(':')
|
65
|
+
[name, (type || 'varchar(255)').downcase.to_sym]
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def init_db_by_csv(csv)
|
70
|
+
header = parser_header(csv.readline)
|
71
|
+
|
72
|
+
cols = header.map { |name, type| "#{name} #{type}" }.join(', ')
|
73
|
+
sql = "CREATE TABLE csv (#{cols});"
|
74
|
+
execute sql
|
75
|
+
|
76
|
+
cache = []
|
77
|
+
col_names = header.map(&:first)
|
78
|
+
csv.each do |line|
|
79
|
+
if cache.length > BATCH_LINES then
|
80
|
+
import_lines(cache, col_names)
|
81
|
+
cache.clear
|
82
|
+
else
|
83
|
+
cache << line.each_with_index.map { |v, i| format_sql_val(v, header[i][1]) }
|
84
|
+
end
|
85
|
+
end
|
86
|
+
import_lines(cache, col_names) unless cache.empty?
|
87
|
+
db
|
88
|
+
end
|
89
|
+
|
90
|
+
def import_lines(lines, col_names)
|
91
|
+
sql = "INSERT INTO csv (#{col_names.join(', ')}) VALUES "
|
92
|
+
values = lines.map { |line| "(#{line.join(',')})" }.join(', ')
|
93
|
+
execute sql + values
|
94
|
+
end
|
95
|
+
|
96
|
+
def format_sql_val(val, type)
|
97
|
+
case type
|
98
|
+
when :int, :integer then val.to_i
|
99
|
+
when :float, :double then val.to_f
|
100
|
+
when :date then "'#{Date.parse(val).to_s}'"
|
101
|
+
when :datetime then "'#{Time.parse(val).strftime('%F %T')}'"
|
102
|
+
else
|
103
|
+
"'#{val.gsub("'", "''")}'"
|
104
|
+
end
|
105
|
+
rescue => e
|
106
|
+
process_sql_error("Parse val: #{val}", e)
|
107
|
+
end
|
108
|
+
|
109
|
+
def process_sql_error(sql, err)
|
110
|
+
$stderr.puts(sql)
|
111
|
+
|
112
|
+
if @error_action == :exit
|
113
|
+
$stderr.puts(e.message)
|
114
|
+
exit
|
115
|
+
else
|
116
|
+
raise err
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
def get_db_path(csv_path)
|
121
|
+
csv_path = csv_path || ''
|
122
|
+
return '' unless File.exist?(csv_path)
|
123
|
+
|
124
|
+
if use_cache
|
125
|
+
stat = File.stat(csv_path)
|
126
|
+
filename = Digest::SHA2.hexdigest(File.absolute_path(csv_path)) + '.cache'
|
127
|
+
file_stat = [File.absolute_path(csv_path), stat.size, stat.ctime].join("\n")
|
128
|
+
stat_path = File.join(CACHE_DIR, filename.gsub(/\.cache$/, '.stat'))
|
129
|
+
cache_path = File.join(CACHE_DIR, filename)
|
130
|
+
|
131
|
+
if File.exist?(stat_path)
|
132
|
+
if File.read(stat_path) == file_stat
|
133
|
+
cache_path
|
134
|
+
else
|
135
|
+
FileUtils.rm(cache_path)
|
136
|
+
cache_path
|
137
|
+
end
|
138
|
+
else
|
139
|
+
File.write(stat_path, file_stat)
|
140
|
+
cache_path
|
141
|
+
end
|
142
|
+
else
|
143
|
+
''
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
data/lib/csvsql.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csvsql/version"
|
4
|
+
|
5
|
+
require 'csv'
|
6
|
+
require 'sqlite3'
|
7
|
+
|
8
|
+
require 'csvsql/db'
|
9
|
+
|
10
|
+
module Csvsql
|
11
|
+
def self.execute(sql, csv_data, opts = {})
|
12
|
+
csvdb = Csvsql::Db.new(opts)
|
13
|
+
csvdb.import(csv_data)
|
14
|
+
pst = csvdb.prepare(sql)
|
15
|
+
CSV.generate do |csv|
|
16
|
+
csv << pst.columns.zip(pst.types).map { |c| c.compact.join(':') }
|
17
|
+
pst.each { |line| csv << line }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
metadata
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: csvsql
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- jiangzhi.xie
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-07-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: sqlite3
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.3.13
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.3.13
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.16.a
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 1.16.a
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: pry
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rubocop
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: 0.52.1
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.52.1
|
83
|
+
description: Process csv with SQL.
|
84
|
+
email:
|
85
|
+
- xiejiangzhi@gmail.com
|
86
|
+
executables:
|
87
|
+
- csvsql
|
88
|
+
extensions: []
|
89
|
+
extra_rdoc_files: []
|
90
|
+
files:
|
91
|
+
- ".gitignore"
|
92
|
+
- ".rspec"
|
93
|
+
- ".rubocop.yml"
|
94
|
+
- ".travis.yml"
|
95
|
+
- Gemfile
|
96
|
+
- Gemfile.lock
|
97
|
+
- LICENSE.txt
|
98
|
+
- README.md
|
99
|
+
- bin/console
|
100
|
+
- bin/setup
|
101
|
+
- csvsql.gemspec
|
102
|
+
- exe/csvsql
|
103
|
+
- lib/csvsql.rb
|
104
|
+
- lib/csvsql/db.rb
|
105
|
+
- lib/csvsql/version.rb
|
106
|
+
homepage: https://github.com/xiejiangzhi/csvsql
|
107
|
+
licenses:
|
108
|
+
- MIT
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.6.13
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: Process csv with SQL.
|
130
|
+
test_files: []
|