incsv 0.2.2 → 0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +13 -4
- data/exe/incsv +16 -0
- data/lib/incsv.rb +1 -0
- data/lib/incsv/console.rb +1 -0
- data/lib/incsv/console/progress.rb +54 -0
- data/lib/incsv/types.rb +5 -0
- data/lib/incsv/types/decimal.rb +28 -0
- data/lib/incsv/types/integer.rb +12 -0
- data/lib/incsv/version.rb +1 -1
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e29974ceb4604157361a02decca58cb0865d6dd
|
4
|
+
data.tar.gz: b8b2f9fcaa691eadf5110260b11fbc3ca297ec24
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ffb7b812b0cc6a8595a16ae1758c8aab76026ce8b950309536df1f13bf7a65f66ed223cabb69b5d606e3e97d9c79a8dafe05bb9cf9f58db46249a55e2d4cd07
|
7
|
+
data.tar.gz: 8f8e5299af6928a5364477cf1eebd00591d6dffbff8ffc5e61fc54552950075674be2987418c79a3db8442b372a923d84b258e9d3cf0ec7c9c42c6a2aefa7594
|
data/README.md
CHANGED
@@ -1,12 +1,23 @@
|
|
1
1
|
# incsv
|
2
2
|
|
3
3
|
incsv is a tool for quickly interrogating CSV files using SQL and the
|
4
|
-
Ruby programming language.
|
4
|
+
Ruby programming language. It is, in essence, a [REPL][] for CSV files.
|
5
5
|
|
6
6
|
It works by loading the CSV into an [SQLite][] database and then
|
7
7
|
dropping you into an interactive Ruby shell. You can then use the
|
8
|
-
[Sequel][] database library to perform further
|
8
|
+
[Sequel][] database library, along with Ruby code, to perform further
|
9
|
+
exploratory analysis.
|
9
10
|
|
11
|
+
This makes it easy to perform the sorts of queries that are just too
|
12
|
+
complex for Excel, or perform queries on much larger datasets than you
|
13
|
+
ever could in a spreadsheet program.
|
14
|
+
|
15
|
+
incsv has been tested on CSV files hundreds of megabytes in size; thanks
|
16
|
+
to SQLite’s excellent performance, it holds up admirably even at these
|
17
|
+
very large sizes, and there’s no reason why it couldn’t handle larger
|
18
|
+
ones, too.
|
19
|
+
|
20
|
+
[REPL]: https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop
|
10
21
|
[SQLite]: https://www.sqlite.org/
|
11
22
|
[Sequel]: http://sequel.jeremyevans.net/
|
12
23
|
|
@@ -38,8 +49,6 @@ A quick example:
|
|
38
49
|
{:name=>"enhanced targeting card"},
|
39
50
|
{:name=>"Giddyup Buttercup"}]
|
40
51
|
|
41
|
-
[REPL]: https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop
|
42
|
-
|
43
52
|
### The less-quick version
|
44
53
|
|
45
54
|
To use incsv, you essentially just need to point it at a CSV file. It’ll
|
data/exe/incsv
CHANGED
@@ -64,8 +64,24 @@ module InCSV
|
|
64
64
|
database = Database.new(csv_file)
|
65
65
|
|
66
66
|
unless database.table_created? && database.imported?
|
67
|
+
pid = fork do
|
68
|
+
progress = Progress.new(output: $stdout, message: "Importing data...")
|
69
|
+
|
70
|
+
trap("TERM") do
|
71
|
+
progress.clear
|
72
|
+
exit
|
73
|
+
end
|
74
|
+
|
75
|
+
# No progress for files that import basically instantly
|
76
|
+
sleep 2
|
77
|
+
|
78
|
+
progress.indeterminate
|
79
|
+
end
|
80
|
+
|
67
81
|
database.create_table
|
68
82
|
database.import
|
83
|
+
|
84
|
+
Process.kill("TERM", pid)
|
69
85
|
end
|
70
86
|
|
71
87
|
console = Console.new(database.db)
|
data/lib/incsv.rb
CHANGED
@@ -0,0 +1 @@
|
|
1
|
+
require_relative "console/progress"
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require "io/console"
|
2
|
+
|
3
|
+
module InCSV
|
4
|
+
class Progress
|
5
|
+
def initialize(output: $stdout, message: "Working...")
|
6
|
+
@output = output
|
7
|
+
@message = message
|
8
|
+
end
|
9
|
+
|
10
|
+
def indeterminate
|
11
|
+
loop do
|
12
|
+
line
|
13
|
+
sleep 1
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def clear
|
18
|
+
output.print "\r"
|
19
|
+
output.print " " * width
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def width
|
25
|
+
_, columns = $stdout.winsize
|
26
|
+
columns
|
27
|
+
end
|
28
|
+
|
29
|
+
def line
|
30
|
+
@offset ||= 0
|
31
|
+
@offset += 1
|
32
|
+
|
33
|
+
inner_width = width - 3 - message.length
|
34
|
+
units = (width / 4.to_f).ceil
|
35
|
+
|
36
|
+
output.print "\r#{message} |"
|
37
|
+
|
38
|
+
case @offset % 4
|
39
|
+
when 0
|
40
|
+
output.print ("=---" * units)[0...inner_width]
|
41
|
+
when 1
|
42
|
+
output.print ("-=--" * units)[0...inner_width]
|
43
|
+
when 2
|
44
|
+
output.print ("--=-" * units)[0...inner_width]
|
45
|
+
when 3
|
46
|
+
output.print ("---=" * units)[0...inner_width]
|
47
|
+
end
|
48
|
+
|
49
|
+
output.print "|"
|
50
|
+
end
|
51
|
+
|
52
|
+
attr_reader :output, :message
|
53
|
+
end
|
54
|
+
end
|
data/lib/incsv/types.rb
CHANGED
@@ -1,5 +1,10 @@
|
|
1
1
|
require "incsv/column_type"
|
2
2
|
|
3
|
+
# The order that these files are required defines the priority order of
|
4
|
+
# column guessing; they should therefore be required in order from most-
|
5
|
+
# to least-specific (with string always last).
|
3
6
|
require "incsv/types/date"
|
4
7
|
require "incsv/types/currency"
|
8
|
+
require "incsv/types/integer"
|
9
|
+
require "incsv/types/decimal"
|
5
10
|
require "incsv/types/string"
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module InCSV
|
2
|
+
module Types
|
3
|
+
# Represents a fixed-precision decimal number.
|
4
|
+
class Decimal < ColumnType
|
5
|
+
# What type of column to create in the database.
|
6
|
+
def self.for_database
|
7
|
+
"DECIMAL(15,10)"
|
8
|
+
end
|
9
|
+
|
10
|
+
# Returns true if the supplied value is a decimal number or
|
11
|
+
# an integer, since integers are also valid decimals.
|
12
|
+
#
|
13
|
+
# This allows for columns that contain e.g.:
|
14
|
+
#
|
15
|
+
# 123.45
|
16
|
+
# 123
|
17
|
+
# 128.2
|
18
|
+
def match?
|
19
|
+
value.strip.match(/\A[0-9]+\.?([0-9]+)?\z/)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Converts values to BigDecimal format before storage.
|
23
|
+
def self.clean_value(value)
|
24
|
+
BigDecimal(value.strip)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/incsv/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: incsv
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: '0.3'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Rob Miller
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-03-
|
11
|
+
date: 2016-03-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -147,11 +147,15 @@ files:
|
|
147
147
|
- lib/incsv.rb
|
148
148
|
- lib/incsv/column.rb
|
149
149
|
- lib/incsv/column_type.rb
|
150
|
+
- lib/incsv/console.rb
|
151
|
+
- lib/incsv/console/progress.rb
|
150
152
|
- lib/incsv/database.rb
|
151
153
|
- lib/incsv/schema.rb
|
152
154
|
- lib/incsv/types.rb
|
153
155
|
- lib/incsv/types/currency.rb
|
154
156
|
- lib/incsv/types/date.rb
|
157
|
+
- lib/incsv/types/decimal.rb
|
158
|
+
- lib/incsv/types/integer.rb
|
155
159
|
- lib/incsv/types/string.rb
|
156
160
|
- lib/incsv/version.rb
|
157
161
|
homepage: https://github.com/robmiller/incsv
|