slyce 1.3.4 → 1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -0
  3. data/bin/slyce +70 -63
  4. data/bin/slyce3 +79 -49
  5. data/bin/slyced +56 -48
  6. data/slyce.gemspec +2 -2
  7. metadata +7 -7
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9e63de054e38ac6dd6b847b8aec37b750955c285c710bf8cc713c61c3e309b78
4
- data.tar.gz: 6a1222e3f162739c198eb83084a6b73688a6acface1d094477884da68f797e6c
3
+ metadata.gz: f8350b63046aa4e84fd9f04bd73ed92140ec3b569ac6c997f53f74e2235ec685
4
+ data.tar.gz: 34ee0a6665542b55df24b55d22150017610f1fdb891860e95b33b8ab30b03eaa
5
5
  SHA512:
6
- metadata.gz: 176b6aa4493b0e10a0a122d414e2924fe764b18016076b1bb0ecf95127cc4927fc8fc946e878232ab5e4e634a054e095f36fa0467bae6381ee7197c12b8e64a2
7
- data.tar.gz: 24732fe38c896f45b6322594af7c55ed3c55e863029ba65146157bf4196e009c032d5d94e8dc68aacdfd19040fbec15b6979516ef1421b4924c7c887708c393c
6
+ metadata.gz: c405dc17703ea8c99617183ef95185deb3703c98ab029106ac87a6e5ea2e3af833c3a6534e99960892de2cd6c6e69ebd2a3653e3c38d93c9c7fc60839df353bd
7
+ data.tar.gz: 25273ba52a3fec83f83da1e867255bbf98d238925d50a18a10f77f1587889e7123e423ac6e27eb14a3cad8a5f7d609b52661285e6a7621365c15d8e98228c23f
data/README.md CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  Ruby utility to show summary statistics or export data from MySQL, SQLite, or DuckDB.
4
4
 
5
+ ## TODO
6
+
7
+ * Unify each each executable to have as close to the same features as possible
8
+
5
9
  ## Usage
6
10
 
7
11
  ```
data/bin/slyce CHANGED
@@ -7,11 +7,36 @@ require "optparse"
7
7
 
8
8
  trap("INT" ) { abort "\n" }
9
9
 
10
- dbas = nil
11
- tabl = nil
10
+ # ==[ Helpers ]==
11
+
12
+ class Mysql2::Client
13
+ alias_method :sql, :query
14
+
15
+ def sql!(stmt, *args, **opts, &block)
16
+ puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
17
+ sql(stmt, *args, **opts, &block)
18
+ end
19
+ end
20
+
21
+ def display(name, data, show, uniq, tots)
22
+ seen = data.inject(0) {|seen, coun| seen += coun[0] }
23
+ rows = [data.size, seen].min
24
+ wide = tots.to_s.size
25
+ fill = " " * wide
26
+ line = "=" * name.size
27
+
28
+ puts "\n#{fill} #{name}\n#{fill} #{line}\n"
29
+ data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
30
+ puts "#{fill} -----\n"
31
+ puts "%*d shown (top %d)" % [wide, seen, rows] if show && rows > 1
32
+ puts "%*d total (all %d)" % [wide, tots, uniq] if uniq > 1
33
+ puts "%*d total" % [wide, tots ] unless uniq > 1
34
+ end
35
+
36
+ # ==[ Options ]==
12
37
 
13
38
  OptionParser.new.instance_eval do
14
- @version = "1.3.4"
39
+ @version = "1.5.0"
15
40
  @banner = "usage: #{program_name} [options] <database> <table>"
16
41
 
17
42
  on "--csv" , "Output comma separated values"
@@ -21,9 +46,9 @@ OptionParser.new.instance_eval do
21
46
  on "-c", "--columns" , "Display column names and quit"
22
47
  on "-d", "--dump" , "Dump database schema and quit"
23
48
  on "-h", "--help" , "Show help and command usage" do Kernel.abort to_s; end
49
+ on "-H", "--no-headers" , "Do not show headers when exporting delimited files"
24
50
  on "-n", "--natural" , "Sort naturally, not numerically"
25
- on "-r", "--rows <count>" , "Rows of data to show", Integer
26
- on "-s", "--suppress" , "Suppress header when exporting delimited files"
51
+ on "-s", "--show <count>" , "Show this many values", Integer
27
52
  on "-t", "--tables" , "Display table names and quit"
28
53
  on "-v", "--version" , "Show version number" do Kernel.abort "#{program_name} #{@version}"; end
29
54
  on "-w", "--where <cond>" , "Where clause (eg - 'age>50 and state='AZ')"
@@ -32,50 +57,27 @@ OptionParser.new.instance_eval do
32
57
  self
33
58
  end.parse!(into: opts={}) rescue abort($!.message)
34
59
 
60
+ dbas = nil
61
+ tabl = nil
62
+
35
63
  xcsv = opts[:csv]
36
64
  xpsv = opts[:psv]
37
65
  xtsv = opts[:tsv]
38
66
  xprt = xcsv || xpsv || xtsv and require "censive"
39
67
 
40
- asky = opts[:ascii ] and require "any_ascii"
41
- dump = opts[:dump]
42
- want = opts[:extract ].to_s.downcase.split(",")
43
- natu = opts[:natural ]
44
- show = opts[:rows ]
45
- hide = opts[:suppress]
46
- filt = opts[:where ] and filt = "\nwhere\n #{filt}"
68
+ asky = opts[:ascii ] and require "any_ascii"
69
+ dump = opts[:dump ]
70
+ want = opts[:extract ].to_s.downcase.split(",")
71
+ natu = opts[:natural ]
72
+ show = opts[:show ]
73
+ hide = opts[:"no-headers"]
74
+ filt = opts[:where ] and filt = "\nwhere\n #{filt}"
47
75
 
48
76
  dbas ||= ARGV.shift or abort "no database given"
49
77
  tabl ||= ARGV.shift or opts[:tables] or !want.empty? or abort "no table given"
50
78
 
51
79
  [xcsv, xpsv, xtsv].compact.size > 1 and abort "only one of csv, psv, or tsv allowed"
52
80
 
53
- # ==[ Helpers ]==
54
-
55
- class Mysql2::Client
56
- alias_method :sql, :query
57
-
58
- def sql!(stmt, *args, **opts, &block)
59
- puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
60
- sql(stmt, *args, **opts, &block)
61
- end
62
- end
63
-
64
- def display(name, data, show, uniq, tots)
65
- seen = data.inject(0) {|seen, coun| seen += coun[0] }
66
- rows = [data.size, seen].min
67
- wide = tots.to_s.size
68
- fill = " " * wide
69
- line = "=" * name.size
70
-
71
- puts "\n#{fill} #{name}\n#{fill} #{line}\n"
72
- data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
73
- puts "#{fill} -----\n"
74
- puts "%*d shown (top %d)" % [wide, seen, rows] if show && rows > 1
75
- puts "%*d total (all %d)" % [wide, tots, uniq] if uniq > 1
76
- puts "%*d total" % [wide, tots ] unless uniq > 1
77
- end
78
-
79
81
  # ==[ Let 'er rip! ]==
80
82
 
81
83
  # get database connection
@@ -84,14 +86,13 @@ if !dbas.include?("/")
84
86
  else
85
87
  dbas = $' if dbas =~ %r|^mysql://| # drop mysql:// prefix, if present
86
88
  auth, dbas = dbas.split("/", 2)
87
- if auth =~ /^(?:(\w+)(?::([^@]+))?@)?(?:([^:]+)?(?::(\d+))?)$/
88
- user, pass, host, port = $1, $2, $3, $4 # user:pass@host:port
89
+ if auth =~ /^(?:(\w+)(?::([^@]+))?@?)?(?:([^:]+)?(?::(\d+))?)$/
89
90
  conf = {
90
91
  database: dbas,
91
92
  username: $1,
92
93
  password: $2,
93
- host: $3,
94
- port: $4,
94
+ host: $3 || "127.0.0.1",
95
+ port: $4 || "3306",
95
96
  }.compact
96
97
  else
97
98
  abort "invalid database value #{dbas.inspect}"
@@ -100,6 +101,7 @@ end
100
101
 
101
102
  # connect to database and get server version
102
103
  conn = Mysql2::Client.new(**conf, as: :array)
104
+ conn.sql("set session sql_mode='ansi'") # ANSI double-quotes
103
105
  ver5 = conn.server_info[:version] =~ /^5/
104
106
 
105
107
  # dump database schema or show table names
@@ -113,11 +115,11 @@ if tabl.nil? || opts[:tables] || opts[:dump]
113
115
  # dump database schema
114
116
  if opts[:dump]
115
117
  pict = "%Y-%m-%dT%H:%M:%S%z"
116
- puts "-- Dump of `#{dbas}` database on #{Time.now.strftime(pict)}\n\n"
118
+ puts "-- Dump of \"#{dbas}\" database on #{Time.now.strftime(pict)}\n\n"
117
119
  puts "set foreign_key_checks=0;\n\n" unless want.empty?
118
120
  tail = []
119
121
  want.each do |name|
120
- text = conn.sql("show create table `#{name}`").to_a.flatten[1] + ";\n\n"
122
+ text = conn.sql("show create table \"#{name}\"").to_a.flatten[1] + ";\n\n"
121
123
  if text =~ /^create table/i
122
124
  puts text
123
125
  elsif text.gsub!(/^(create ).*?(?=view)/i, '\1')
@@ -135,7 +137,7 @@ if tabl.nil? || opts[:tables] || opts[:dump]
135
137
  end
136
138
 
137
139
  # get column names
138
- resu = conn.sql("select * from `#{tabl}` limit 0")
140
+ resu = conn.sql("select * from \"#{tabl}\" limit 0")
139
141
  cols = resu.fields
140
142
  want = want.empty? ? cols : want.select {|e| cols.include?(e) }
141
143
 
@@ -150,20 +152,23 @@ end
150
152
 
151
153
  # handle exports
152
154
  if xprt
153
- list = want.map {|item| "`#{item}`" }.join(", ")
154
- stmt = show ? "limit #{show}" : ""
155
- data = conn.sql(<<~"" + stmt).to_a
155
+ list = want.map {|item| %{"#{item}"} }.join(", ")
156
+ limt = show ? "limit #{show}" : ""
157
+ data = conn.sql(<<~"".rstrip).to_a
156
158
  select
157
159
  #{list}
158
160
  from
159
- `#{tabl}`#{filt}
161
+ "#{tabl}"
162
+ #{filt}
163
+ #{limt}
160
164
 
161
165
  seps = xcsv ? "," : xtsv ? "\t" : xpsv ? "|" : abort("unknown separator #{seps.inspect}")
162
166
 
163
167
  Censive.write(sep: seps) do |csv|
164
168
  csv << want unless hide
165
169
  data.each do |row|
166
- csv << row.map {|e| asky ? AnyAscii.transliterate(e.to_s) : e.to_s }
170
+ # csv << row.map {|e| asky ? AnyAscii.transliterate(e.to_s) : e.to_s }
171
+ csv << row.map {|e| asky ? AnyAscii.transliterate(e.to_s) : e.nil? ? nil : e.to_s }
167
172
  end
168
173
  end
169
174
 
@@ -172,34 +177,36 @@ end
172
177
 
173
178
  want.each do |name|
174
179
  sort = natu ? "" : "cnt desc,"
180
+ limt = show ? "limit #{show}" : ""
175
181
  like =(ver5 ? <<~"" : <<~"").gsub(/(.)^/m, '\1 ').rstrip
176
- -if((`#{name}` rlike '^[-+]?((0|([1-9][0-9]*)(\\\\.[0-9]*)?)|((0|([1-9][0-9]*))\\\\.[0-9]+))$'), `#{name}` + 0, null) desc,
177
- -if((`#{name}` rlike '^0[0-9]+$'), length(`#{name}`), null) desc,
178
- -if((`#{name}` rlike '^[0-9]'), length(concat("1", `#{name}`) + 0), null) desc,
182
+ -if(("#{name}" rlike '^[-+]?((0|([1-9][0-9]*)(\\\\.[0-9]*)?)|((0|([1-9][0-9]*))\\\\.[0-9]+))$'), "#{name}" + 0, null) desc,
183
+ -if(("#{name}" rlike '^0[0-9]+$'), length("#{name}"), null) desc,
184
+ -if(("#{name}" rlike '^[0-9]'), length(concat('1', "#{name}") + 0), null) desc,
179
185
 
180
- -if(regexp_like(`#{name}`, '^[-+]?((0|([1-9]\\\\d*)(\\\\.\\\\d*)?)|((0|([1-9]\\\\d*))\\\\.\\\\d+))$'), `#{name}` + 0, null) desc,
181
- -if(regexp_like(`#{name}`, '^0\\\\d+$'), length(`#{name}`), null) desc,
182
- -if(regexp_like(`#{name}`, '^\\\\d'), regexp_instr(`#{name}`, '[^\\\\d]'), null) desc,
186
+ -if(regexp_like("#{name}", '^[-+]?((0|([1-9]\\\\d*)(\\\\.\\\\d*)?)|((0|([1-9]\\\\d*))\\\\.\\\\d+))$'), "#{name}" + 0, null) desc,
187
+ -if(regexp_like("#{name}", '^0\\\\d+$'), length("#{name}"), null) desc,
188
+ -if(regexp_like("#{name}", '^\\\\d'), regexp_instr("#{name}", '[^\\\\d]'), null) desc,
183
189
 
184
190
  data = conn.sql(<<~"".rstrip).to_a
185
191
  select
186
192
  count(*) as cnt,
187
- `#{name}` as val
193
+ "#{name}" as val
188
194
  from
189
- `#{tabl}`#{filt}
195
+ "#{tabl}"#{filt}
190
196
  group by
191
197
  val
192
198
  order by #{sort}
193
199
  #{like}
194
- `#{name}` is null, `#{name}`
195
- #{show ? "limit #{show}" : ""}
200
+ "#{name}" is null, "#{name}"
201
+ #{limt}
196
202
 
197
203
  uniq, tots = conn.sql(<<~"".rstrip).to_a[0]
198
204
  select
199
- count(distinct(ifnull(`#{name}`,0))),
200
- count(ifnull(`#{name}`,0))
205
+ count(distinct(ifnull("#{name}",0))),
206
+ count(ifnull("#{name}",0))
201
207
  from
202
- `#{tabl}`#{filt}
208
+ "#{tabl}"
209
+ #{filt}
203
210
 
204
211
  display(name, data, show, uniq, tots)
205
212
  end
data/bin/slyce3 CHANGED
@@ -6,7 +6,7 @@
6
6
  #
7
7
  # For example, on Apple Silicon with macOS with an M1 you can use:
8
8
  #
9
- # wget https://github.com/nalgeon/sqlean/releases/download/0.19.3/sqlean-macos-arm64.zip
9
+ # wget https://github.com/nalgeon/sqlean/releases/download/0.21.8/sqlean-macos-arm64.zip
10
10
  # unzip sqlean-macos-arm64.zip regexp.dylib
11
11
 
12
12
  STDOUT.sync = true
@@ -16,43 +16,14 @@ require "optparse"
16
16
 
17
17
  trap("INT" ) { abort "\n" }
18
18
 
19
- dbas = nil
20
- tabl = nil
21
-
22
- OptionParser.new.instance_eval do
23
- @banner = "usage: #{program_name} [options] <database> <table>"
24
-
25
- on "-c", "--columns" , "Display column names and quit"
26
- on "-h", "--help" , "Show help and command usage" do Kernel.abort to_s; end
27
- on "-n", "--natural" , "Sort naturally, not numerically"
28
- on "-r", "--regexp <path>" , "Path to the sqlean/regexp extension"
29
- on "-s", "--show <count>" , "Show this many values", Integer
30
- on "-v", "--version" , "Show version number" do Kernel.abort "#{program_name} #{VERSION}"; end
31
- on "-w", "--where <cond>" , "Where clause (eg - 'age>50 and state='AZ')"
32
- on "-x", "--extract <col1,col2,...>", "Comma separated list of columns to extract"
33
-
34
- self
35
- end.parse!(into: opts={}) rescue abort($!.message)
36
-
37
- filt = opts[:where] and filt = "where\n #{filt}"
38
- natu = opts[:natural]
39
- regx = opts[:regexp] || Dir["{.,sqlean}/regexp.{dll,dylib,so}"].first
40
- show = opts[:show]
41
- want = opts[:extract].to_s.downcase.split(",")
42
-
43
- dbas ||= ARGV.shift or abort "no database given"
44
- tabl ||= ARGV.shift or abort "no table given"
45
-
46
- regx && File.exist?(regx) or abort "no regexp extension found#{regx ? " at '#{regx}'" : ''}"
47
-
48
19
  # ==[ Helpers ]==
49
20
 
50
21
  class Extralite::Database
51
22
  alias_method :sql, :query_ary
52
23
 
53
- def sql!(stmt, *args, **, &)
24
+ def sql!(stmt, *args, **opts, &block)
54
25
  puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
55
- sql(stmt, *args, **, &)
26
+ sql(stmt, *args, **opts, &block)
56
27
  end
57
28
  end
58
29
 
@@ -61,22 +32,76 @@ def display(name, data, show, uniq, tots)
61
32
  rows = [data.size, seen].min
62
33
  wide = tots.to_s.size
63
34
  fill = " " * wide
35
+ over = "\n#{fill} "
64
36
  line = "=" * name.size
65
37
 
66
38
  puts "\n#{fill} #{name}\n#{fill} #{line}\n"
67
- data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
39
+ # data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
40
+ data.each do |cnt, val| # TODO: only enable this with an option? (it's rarely useful)
41
+ puts "%*d %s" % [wide, cnt, val&.gsub("\n", over) || "NULL"]
42
+ end
68
43
  puts "#{fill} -----\n"
69
- puts "%*d shown (top %d)" % [wide, seen, rows] if show and rows > 1
44
+ puts "%*d shown (top %d)" % [wide, seen, rows] if show && rows > 1
70
45
  puts "%*d total (all %d)" % [wide, tots, uniq] if uniq > 1
71
46
  puts "%*d total" % [wide, tots ] unless uniq > 1
72
47
  end
73
48
 
49
+ # ==[ Options ]==
50
+
51
+ OptionParser.new.instance_eval do
52
+ @version = "1.5.0"
53
+ @banner = "usage: #{program_name} [options] <database> <table>"
54
+
55
+ on "-c", "--columns" , "Display column names and quit"
56
+ on "-d", "--delete" , "Delete the .slyce database first"
57
+ on "-h", "--help" , "Show help and command usage" do Kernel.abort to_s; end
58
+ on "-k", "--keep" , "For CSV files, keep the .slyce database for reuse"
59
+ on "-n", "--natural" , "Sort naturally, not numerically"
60
+ on "-r", "--regexp <path>" , "Path to the sqlean/regexp extension"
61
+ on "-s", "--show <count>" , "Show this many values", Integer
62
+ on "-v", "--version" , "Show version number" do Kernel.abort "#{program_name} #{@version}"; end
63
+ on "-w", "--where <cond>" , "Where clause (eg - 'age>50 and state='AZ')"
64
+ on "-x", "--extract <a,b,c...>" , "Comma separated list of columns to extract"
65
+
66
+ self
67
+ end.parse!(into: opts={}) rescue abort($!.message)
68
+
69
+ dbas = nil
70
+ tabl = nil
71
+
72
+ nuke = opts[:delete ]
73
+ want = opts[:extract ].to_s.downcase.split(",")
74
+ keep = opts[:keep ]
75
+ natu = opts[:natural ]
76
+ regx = opts[:regexp ] || Dir["{.,sqlean,#{ENV['HOME']}}/regexp.{dll,dylib,so}"].first
77
+ show = opts[:show ]
78
+ filt = opts[:where ] and filt = "\nwhere\n #{filt}"
79
+
80
+ # ensure regexp extension is available
81
+ regx && File.exist?(regx) or abort "no regexp extension found#{regx ? " at '#{regx}'" : ''}"
82
+
83
+ # eager deletion of prior .slyce database
84
+ nuke and `rm -f .slyce`
85
+
86
+ dbas ||= ARGV.shift or nuke ? exit : abort("no database given")
87
+
88
+ case dbas
89
+ when /(\.csv)$/, "/dev/stdin", "-"
90
+ file = $1 ? dbas : "-"
91
+ dbas = ".slyce"
92
+ tabl = "csv"
93
+ `rm -f "#{dbas}"` if File.exist?(dbas) && !keep
94
+ `sqlite3 -csv '#{dbas}' ".import '|cat #{file}' '#{tabl}'"`
95
+ else
96
+ tabl ||= ARGV.shift or abort "no table given"
97
+ end
98
+
74
99
  # ==[ Let 'er rip! ]==
75
100
 
76
101
  conn = Extralite::Database.new(dbas)
77
102
  resu = conn.load_extension(regx) rescue abort("unable to load regexp extension '#{regx}'")
78
- cols = conn.columns("select * from `#{tabl}` limit 0").map(&:to_s)
79
- want = want.empty? ? cols : want & cols
103
+ cols = conn.columns("select * from \"#{tabl}\" limit 0").map(&:to_s)
104
+ want = want.empty? ? cols : Hash[cols.map(&:downcase).zip(cols)].values_at(*want).compact
80
105
 
81
106
  if opts[:columns]
82
107
  puts cols
@@ -89,30 +114,35 @@ end
89
114
 
90
115
  want.each do |name|
91
116
  sort = natu ? "" : "cnt desc,"
92
- stmt = show ? "limit #{show}" : ""
93
- data = conn.sql(<<~"" + stmt).to_a
117
+ limt = show ? "limit #{show}" : ""
118
+ like = <<~"".gsub(/(.)^/m, '\1 ').rstrip
119
+ -iif(regexp_like("#{name}", '^[-+]?((0|([1-9]\\d*)(\\.\\d*)?)|((0|([1-9]\\d*))\\.\\d+))$'), "#{name}" + 0, null) desc,
120
+ -iif(regexp_like("#{name}", '^0\\d+$'), length("#{name}"), null) desc,
121
+ -iif(regexp_like("#{name}", '^\\d'), length(regexp_substr("#{name}", '^\\d+')), null) desc,
122
+
123
+ data = conn.sql(<<~"".rstrip).to_a
94
124
  select
95
125
  count(*) as cnt,
96
- `#{name}` as val
126
+ "#{name}" as val
97
127
  from
98
- `#{tabl}`
99
- #{filt}
128
+ "#{tabl}"#{filt}
100
129
  group by
101
130
  val
102
131
  order by #{sort}
103
- -iif(regexp_like(`#{name}`, '^[-+]?((0|([1-9]\\d*)(\\.\\d*)?)|((0|([1-9]\\d*))\\.\\d+))$'), `#{name}` + 0, null) desc,
104
- -iif(regexp_like(`#{name}`, '^0\\d+$'), length(`#{name}`), null) desc,
105
- -iif(regexp_like(`#{name}`, '^\\d'), length(regexp_substr(`#{name}`, '^\\d+')), null) desc,
106
- `#{name}` is null, `#{name}`
132
+ #{like}
133
+ "#{name}" is null, "#{name}"
107
134
  collate nocase
135
+ #{limt}
108
136
 
109
- uniq, tots = conn.sql(<<~"").to_a[0]
137
+ uniq, tots = conn.sql(<<~"".rstrip).to_a[0]
110
138
  select
111
- count(distinct(ifnull(`#{name}`,0))),
112
- count(ifnull(`#{name}`,0))
139
+ count(distinct(ifnull("#{name}",0))),
140
+ count(ifnull("#{name}",0))
113
141
  from
114
- `#{tabl}`
142
+ "#{tabl}"
115
143
  #{filt}
116
144
 
117
145
  display(name, data, show, uniq, tots)
118
146
  end
147
+
148
+ `rm -f "#{dbas}"` if file && !keep
data/bin/slyced CHANGED
@@ -7,11 +7,38 @@ require "optparse"
7
7
 
8
8
  trap("INT" ) { abort "\n" }
9
9
 
10
- dbas = nil
11
- tabl = nil
10
+ # ==[ Helpers ]==
11
+
12
+ DuckDB::Result.use_chunk_each = true
13
+
14
+ class DuckDB::Connection
15
+ alias_method :sql, :query
16
+
17
+ def sql!(stmt, *args, **opts, &block)
18
+ puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
19
+ sql(stmt, *args, **opts, &block)
20
+ end
21
+ end
22
+
23
+ def display(name, data, show, uniq, tots)
24
+ seen = data.inject(0) {|seen, coun| seen += coun[0] }
25
+ rows = [data.size, seen].min
26
+ wide = tots.to_s.size
27
+ fill = " " * wide
28
+ line = "=" * name.size
29
+
30
+ puts "\n#{fill} #{name}\n#{fill} #{line}\n"
31
+ data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
32
+ puts "#{fill} -----\n"
33
+ puts "%*d shown (top %d)" % [wide, seen, rows] if show && rows > 1
34
+ puts "%*d total (all %d)" % [wide, tots, uniq] if uniq > 1
35
+ puts "%*d total" % [wide, tots ] unless uniq > 1
36
+ end
37
+
38
+ # ==[ Options ]==
12
39
 
13
40
  OptionParser.new.instance_eval do
14
- @version = "1.3.1"
41
+ @version = "1.5.0"
15
42
  @banner = "usage: #{program_name} [options] <database> <table>"
16
43
 
17
44
  on "--csv" , "Output comma separated values"
@@ -20,9 +47,9 @@ OptionParser.new.instance_eval do
20
47
  on "-a", "--ascii" , "Convert data to ASCII using AnyAscii"
21
48
  on "-c", "--columns" , "Display column names and quit"
22
49
  on "-h", "--help" , "Show help and command usage" do Kernel.abort to_s; end
50
+ on "-H", "--no-headers" , "Do not show headers when exporting delimited files"
23
51
  on "-n", "--natural" , "Sort naturally, not numerically"
24
- on "-r", "--rows <count>" , "Rows of data to show", Integer
25
- on "-s", "--suppress" , "Suppress header when exporting delimited files"
52
+ on "-s", "--show <count>" , "Show this many values", Integer
26
53
  on "-v", "--version" , "Show version number" do Kernel.abort "#{program_name} #{@version}"; end
27
54
  on "-w", "--where <cond>" , "Where clause (eg - 'age>50 and state='AZ')"
28
55
  on "-x", "--extract <a,b,c...>" , "Comma separated list of columns to extract"
@@ -30,49 +57,26 @@ OptionParser.new.instance_eval do
30
57
  self
31
58
  end.parse!(into: opts={}) rescue abort($!.message)
32
59
 
60
+ dbas = nil
61
+ tabl = nil
62
+
33
63
  xcsv = opts[:csv]
34
64
  xpsv = opts[:psv]
35
65
  xtsv = opts[:tsv]
36
66
  xprt = xcsv || xpsv || xtsv and require "censive"
37
67
 
38
- asky = opts[:ascii ] and require "any_ascii"
39
- filt = opts[:where ] and filt = "where\n #{filt}"
40
- hide = opts[:suppress]
41
- natu = opts[:natural ]
42
- show = opts[:rows ]
43
- want = opts[:extract].to_s.downcase.split(",")
68
+ asky = opts[:ascii ] and require "any_ascii"
69
+ want = opts[:extract] .to_s.downcase.split(",")
70
+ natu = opts[:natural ]
71
+ show = opts[:show ]
72
+ hide = opts[:"no-headers"]
73
+ filt = opts[:where ] and filt = "\nwhere\n #{filt}"
44
74
 
45
75
  dbas ||= ARGV.shift or abort "no database given"
46
76
  tabl ||= ARGV.shift or abort "no table given"
47
77
 
48
78
  [xcsv, xpsv, xtsv].compact.size > 1 and abort "only one of csv, psv, or tsv allowed"
49
79
 
50
- # ==[ Helpers ]==
51
-
52
- class DuckDB::Connection
53
- alias_method :sql, :query
54
-
55
- def sql!(stmt, *args, **opts, &block)
56
- puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
57
- sql(stmt, *args, **opts, &block)
58
- end
59
- end
60
-
61
- def display(name, data, show, uniq, tots)
62
- seen = data.inject(0) {|seen, coun| seen += coun[0] }
63
- rows = [data.size, seen].min
64
- wide = tots.to_s.size
65
- fill = " " * wide
66
- line = "=" * name.size
67
-
68
- puts "\n#{fill} #{name}\n#{fill} #{line}\n"
69
- data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
70
- puts "#{fill} -----\n"
71
- puts "%*d shown (top %d)" % [wide, seen, rows] if show && rows > 1
72
- puts "%*d total (all %d)" % [wide, tots, uniq] if uniq > 1
73
- puts "%*d total" % [wide, tots ] unless uniq > 1
74
- end
75
-
76
80
  # ==[ Let 'er rip! ]==
77
81
 
78
82
  conn = DuckDB::Database.open(dbas).connect
@@ -96,14 +100,15 @@ end
96
100
 
97
101
  # handle exports
98
102
  if xprt
99
- list = want.map {|item| "\"#{item}\"" }.join(", ")
100
- stmt = show ? "limit #{show}" : ""
101
- data = conn.sql(<<~"" + stmt).to_a
103
+ list = want.map {|item| %{"#{item}"} }.join(", ")
104
+ limt = show ? "limit #{show}" : ""
105
+ data = conn.sql(<<~"".rstrip).to_a
102
106
  select
103
107
  #{list}
104
108
  from
105
109
  "#{tabl}"
106
110
  #{filt}
111
+ #{limt}
107
112
 
108
113
  seps = xcsv ? "," : xtsv ? "\t" : xpsv ? "|" : abort("unknown separator #{seps.inspect}")
109
114
 
@@ -119,23 +124,26 @@ end
119
124
 
120
125
  want.each do |name|
121
126
  sort = natu ? "" : "cnt desc,"
122
- stmt = show ? "limit #{show}" : ""
123
- data = conn.sql(<<~"" + stmt).to_a
127
+ limt = show ? "limit #{show}" : ""
128
+ like = <<~"".gsub(/(.)^/m, '\1 ').rstrip
129
+ if(regexp_matches("#{name}", '^[-+]?((0|([1-9]\\d*)(\\.\\d*)?)|((0|([1-9]\\d*))\\.\\d+))$'),cast("#{name}" as double),null) nulls last,
130
+ if(regexp_matches("#{name}", '^0\\d*$'),length("#{name}"),null) nulls last,
131
+ if(regexp_matches("#{name}", '^\\d+\\D'),length(regexp_extract("#{name}",'^(\\d+)',1)),null) nulls last,
132
+
133
+ data = conn.sql(<<~"".rstrip).to_a
124
134
  select
125
135
  count(*) as cnt,
126
136
  "#{name}" as val
127
137
  from
128
- "#{tabl}"
129
- #{filt}
138
+ "#{tabl}"#{filt}
130
139
  group by
131
140
  val
132
141
  order by #{sort}
133
- if(regexp_matches("#{name}", '^[-+]?((0|([1-9]\\d*)(\\.\\d*)?)|((0|([1-9]\\d*))\\.\\d+))$'),cast("#{name}" as double),null) nulls last,
134
- if(regexp_matches("#{name}", '^0\\d*$'),length("#{name}"),null) nulls last,
135
- if(regexp_matches("#{name}", '^\\d+\\D'),length(regexp_extract("#{name}",'^(\\d+)',1)),null) nulls last,
142
+ #{like}
136
143
  "#{name}" is null, "#{name}"
144
+ #{limt}
137
145
 
138
- uniq, tots = conn.sql(<<~"").to_a[0]
146
+ uniq, tots = conn.sql(<<~"".rstrip).to_a[0]
139
147
  select
140
148
  count(distinct(ifnull("#{name}",0))),
141
149
  count(ifnull("#{name}",0))
data/slyce.gemspec CHANGED
@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
13
13
  s.executables = `cd bin && git ls-files .`.split("\n")
14
14
  s.add_runtime_dependency "any_ascii", "~> 0.3.2"
15
15
  s.add_runtime_dependency "censive", "~> 1.1.0"
16
- s.add_runtime_dependency "duckdb", "~> 0.8.0"
17
- s.add_runtime_dependency "extralite-bundle", "~> 1.25"
16
+ s.add_runtime_dependency "duckdb", "~> 0.8.1"
17
+ s.add_runtime_dependency "extralite-bundle", "~> 2.1"
18
18
  s.add_runtime_dependency "mysql2", "~> 0.5"
19
19
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: slyce
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.4
4
+ version: 1.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Steve Shreeve
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-06-30 00:00:00.000000000 Z
11
+ date: 2023-10-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: any_ascii
@@ -44,28 +44,28 @@ dependencies:
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: 0.8.0
47
+ version: 0.8.1
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: 0.8.0
54
+ version: 0.8.1
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: extralite-bundle
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '1.25'
61
+ version: '2.1'
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '1.25'
68
+ version: '2.1'
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: mysql2
71
71
  requirement: !ruby/object:Gem::Requirement
@@ -115,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
115
115
  - !ruby/object:Gem::Version
116
116
  version: '0'
117
117
  requirements: []
118
- rubygems_version: 3.4.14
118
+ rubygems_version: 3.4.20
119
119
  signing_key:
120
120
  specification_version: 4
121
121
  summary: Ruby utility to show data statistics for MySQL databases