slyce 1.0.2 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6b0933ed29b724b0cca2ed1a47ae8f18afd875d01957d8497a1da5e64aea67f
-  data.tar.gz: 25e01d18ac4d15c989c8cfdb3bce0af15b7d66c176e8d411bd6634e263283d27
+  metadata.gz: 88c8c083ec80e6f7e25a3ba665a4bc9ec578b0f7d3939298b4ce97a1a13c430d
+  data.tar.gz: 0a4af788b109e4b8f000d304c53d7d1ad2b090545eca213bf9f7901cc40b1188
 SHA512:
-  metadata.gz: a9ea1296cd1f3bd8d946e9a389b57f604771dfe960c6c2181128f4460dae0ae854ae8927142649c0d5ad7b45d07da9eef462fb77f2e6131607515540d700027d
-  data.tar.gz: 91bb4e8c118925556fc218486d1e8d944b90ed216da2d84cf38ba2e956d0fc9a2e87e5fccf7e1fa5f4a446159b91fec7c13f000ae9442f15aa66ad9b9411648c
+  metadata.gz: 285327c0dba3c36a4c3da6ff0fa033589094ea8a72b2ffe49fe7370d2c89d137835e2aa60f7e0c40a4449f823e55434166b93fcc14c8bc39d431814a9d38b092
+  data.tar.gz: 78737aefd1bca2982594a175c8ee2ae49a5040b12e80382b57a792df3040789400cc747f6d19ab68eaa88c29a2341d2b679b4e895bf622a5b6df1039b5d0b5f6

data/README.md

@@ -1,2 +1,66 @@
 # slyce
-Ruby utility to show data statistics for MySQL databases
+
+Ruby utility to show summary statistics or export data from MySQL, SQLite, or DuckDB.
+
+## Supported platforms
+
+### MySQL
+
+### SQLite
+
+### DuckDB
+
+Ensure the `duckdb` gem is installed via:
+
+`gem install duckdb`
+
+## Example
+
+The following will open `my-database.duck` and will export the first 10 rows
+with the `last_name` field equal to `Miller` and it will extract six fields
+starting with `id,first_name,...`. The output will be a `csv` file to `STDOUT`.
+
+```slyced my-database.duck members -r 10 -w "last_name='Miller'" -xid,first_name,city,state,zip,medical_plan --csv```
+
+The following will show the most frequent 5 values from the `temp_table` table
+in the `nearsite` database for these fields: `ef_id,external_id,insurance_plan`.
+In addition, only rows with the condition specified in the `-w` option are shown.
+
+```slyce nearsite temp_table -s 10 -w "ef_id>=251 and benefit_status='X'" -x ef_id,external_id,insurance_plan -r 5```
+
+The resulting table looks like:
+
+```text
+    ef_id
+    =====
+781 255
+ 35 253
+ 31 510
+ 19 251
+  5 258
+    -----
+871 shown (top 5)
+879 total (all 10)
+
+    external_id
+    ===========
+816 NULL
+  1 1235
+  1 1266
+  1 1320
+  1 1548
+    -----
+820 shown (top 5)
+879 total (all 64)
+
+    insurance_plan
+    ==============
+176 Aetna PPO
+144 Aetna HSA
+103 NULL
+ 82 Kaiser Northern CA
+ 73
+    -----
+578 shown (top 5)
+879 total (all 35)
+```

data/bin/slyce

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby
 
-VERSION="1.0.2"
-
 STDOUT.sync = true
 
 require "mysql2"
@@ -13,27 +11,42 @@ dbas = nil
 tabl = nil
 
 OptionParser.new.instance_eval do
+  @version = "1.2.0"
   @banner  = "usage: #{program_name} [options] <database> <table>"
 
+  on       "--csv"                    , "Output comma separated values"
+  on       "--psv"                    , "Output pipe separated values"
+  on       "--tsv"                    , "Output tab separated values"
+  on "-a", "--ascii"                  , "Convert data to ASCII using AnyAscii"
   on "-c", "--columns"                , "Display column names and quit"
   on "-h", "--help"                   , "Show help and command usage" do Kernel.abort to_s; end
   on "-n", "--natural"                , "Sort naturally, not numerically"
-  on "-s", "--show <count>"           , "Show this many values", Integer
-  on "-v", "--version"                , "Show version number" do Kernel.abort "#{program_name} #{VERSION}"; end
+  on "-r", "--rows <count>"           , "Rows of data to select", Integer
+  on "-s", "--suppress"               , "Suppress header when exporting delimited files"
+  on "-v", "--version"                , "Show version number" do Kernel.abort "#{program_name} #{@version}"; end
   on "-w", "--where <cond>"           , "Where clause (eg - 'age>50 and state='AZ')"
   on "-x", "--extract <col1,col2,...>", "Comma separated list of columns to extract"
 
   self
 end.parse!(into: opts={}) rescue abort($!.message)
 
-filt = opts[:where] and filt = "where\n  #{filt}"
-natu = opts[:natural]
-show = opts[:show]
+xcsv = opts[:csv]
+xpsv = opts[:psv]
+xtsv = opts[:tsv]
+xprt = xcsv || xpsv || xtsv and require "censive"
+
+asky = opts[:ascii   ] and require "any_ascii"
+filt = opts[:where   ] and filt = "where\n  #{filt}"
+hide = opts[:suppress]
+natu = opts[:natural ]
+show = opts[:rows    ]
 want = opts[:extract].to_s.downcase.split(",")
 
 dbas ||= ARGV.shift or abort "no database given"
 tabl ||= ARGV.shift or abort "no table given"
 
+[xcsv, xpsv, xtsv].compact.size > 1 and abort "only one of csv, psv, or tsv allowed"
+
 # ==[ Helpers ]==
 
 class Mysql2::Client
@@ -76,6 +89,29 @@ if want.empty?
   abort "no columns are selected"
 end
 
+# handle exports
+if xprt
+  list = want.map {|item| "`#{item}`" }.join(", ")
+  stmt = show ? "limit #{show}" : ""
+  data = conn.sql(<<~"" + stmt).to_a
+    select
+      #{list}
+    from
+      `#{tabl}`
+    #{filt}
+
+  seps = xcsv ? "," : xtsv ? "\t" : xpsv ? "|" : abort("unknown separator #{seps.inspect}")
+
+  Censive.writer(sep: seps) do |csv|
+    csv << want unless hide
+    data.each do |row|
+      csv << row.map {|e| asky ? AnyAscii.transliterate(e.to_s) : e.to_s }
+    end
+  end
+
+  exit
+end
+
 want.each do |name|
   sort = natu ? "" : "cnt desc,"
   stmt = show ? "limit #{show}" : ""

data/bin/slyce3

@@ -9,8 +9,6 @@
 #   wget https://github.com/nalgeon/sqlean/releases/download/0.19.3/sqlean-macos-arm64.zip
 #   unzip sqlean-macos-arm64.zip regexp.dylib
 
-VERSION="1.0.2"
-
 STDOUT.sync = true
 
 require "extralite"

data/bin/slyced

@@ -0,0 +1,147 @@
+#!/usr/bin/env ruby
+
+STDOUT.sync = true
+
+require "duckdb"
+require "optparse"
+
+trap("INT" ) { abort "\n" }
+
+dbas = nil
+tabl = nil
+
+OptionParser.new.instance_eval do
+  @version = "1.2.0"
+  @banner  = "usage: #{program_name} [options] <database> <table>"
+
+  on       "--csv"                    , "Output comma separated values"
+  on       "--psv"                    , "Output pipe separated values"
+  on       "--tsv"                    , "Output tab separated values"
+  on "-a", "--ascii"                  , "Convert data to ASCII using AnyAscii"
+  on "-c", "--columns"                , "Display column names and quit"
+  on "-h", "--help"                   , "Show help and command usage" do Kernel.abort to_s; end
+  on "-n", "--natural"                , "Sort naturally, not numerically"
+  on "-r", "--rows <count>"           , "Rows of data to select", Integer
+  on "-s", "--suppress"               , "Suppress header when exporting delimited files"
+  on "-v", "--version"                , "Show version number" do Kernel.abort "#{program_name} #{@version}"; end
+  on "-w", "--where <cond>"           , "Where clause (eg - 'age>50 and state='AZ')"
+  on "-x", "--extract <col1,col2,...>", "Comma separated list of columns to extract"
+
+  self
+end.parse!(into: opts={}) rescue abort($!.message)
+
+xcsv = opts[:csv]
+xpsv = opts[:psv]
+xtsv = opts[:tsv]
+xprt = xcsv || xpsv || xtsv and require "censive"
+
+asky = opts[:ascii   ] and require "any_ascii"
+filt = opts[:where   ] and filt = "where\n  #{filt}"
+hide = opts[:suppress]
+natu = opts[:natural ]
+show = opts[:rows    ]
+want = opts[:extract].to_s.downcase.split(",")
+
+dbas ||= ARGV.shift or abort "no database given"
+tabl ||= ARGV.shift or abort "no table given"
+
+[xcsv, xpsv, xtsv].compact.size > 1 and abort "only one of csv, psv, or tsv allowed"
+
+# ==[ Helpers ]==
+
+class DuckDB::Connection
+  alias_method :sql, :query
+
+  def sql!(stmt, *args, **, &)
+    puts "\n==[ SQL statement ]==\n\n", stmt.strip, ";"
+    sql(stmt, *args, **, &)
+  end
+end
+
+def display(name, data, show, uniq, tots)
+  seen = data.inject(0) {|seen, coun| seen += coun[0] }
+  rows = [data.size, seen].min
+  wide = tots.to_s.size
+  fill = " " * wide
+  line = "=" * name.size
+
+  puts "\n#{fill} #{name}\n#{fill} #{line}\n"
+  data.each {|cnt, val| puts "%*d %s" % [wide, cnt, val || "NULL"] }
+  puts "#{fill} -----\n"
+  puts "%*d shown (top %d)" % [wide, seen, rows] if     show && rows > 1
+  puts "%*d total (all %d)" % [wide, tots, uniq] if     uniq > 1
+  puts "%*d total"          % [wide, tots      ] unless uniq > 1
+end
+
+# ==[ Let 'er rip! ]==
+
+conn = DuckDB::Database.open(dbas).connect
+resu = conn.query(<<~end)
+  select column_name
+  from   information_schema.columns
+  where  table_name='#{tabl}'
+  order  by ordinal_position
+end
+cols = resu.to_a.flatten
+want = want.empty? ? cols : want & cols
+
+if opts[:columns]
+  puts cols
+  exit
+end
+
+if want.empty?
+  abort "no columns are selected"
+end
+
+# handle exports
+if xprt
+  list = want.map {|item| "\"#{item}\"" }.join(", ")
+  stmt = show ? "limit #{show}" : ""
+  data = conn.sql(<<~"" + stmt).to_a
+    select
+      #{list}
+    from
+      "#{tabl}"
+    #{filt}
+
+  seps = xcsv ? "," : xtsv ? "\t" : xpsv ? "|" : abort("unknown separator #{seps.inspect}")
+
+  Censive.writer(sep: seps) do |csv|
+    csv << want unless hide
+    data.each do |row|
+      csv << row.map {|e| asky ? AnyAscii.transliterate(e.to_s) : e.to_s }
+    end
+  end
+
+  exit
+end
+
+want.each do |name|
+  sort = natu ? "" : "cnt desc,"
+  stmt = show ? "limit #{show}" : ""
+  data = conn.sql(<<~"" + stmt).to_a
+    select
+      count(*) as cnt,
+      "#{name}" as val
+    from
+      "#{tabl}"
+    #{filt}
+    group by
+      val
+    order by #{sort}
+      if(regexp_matches("#{name}", '^[-+]?((0|([1-9]\\d*)(\\.\\d*)?)|((0|([1-9]\\d*))\\.\\d+))$'),cast("#{name}" as double),null) nulls last,
+      if(regexp_matches("#{name}", '^0\\d*$'),length("#{name}"),null)                                                             nulls last,
+      if(regexp_matches("#{name}", '^\\d+\\D'),length(regexp_extract("#{name}",'^(\\d+)',1)),null)                                nulls last,
+      "#{name}" is null, "#{name}"
+
+  uniq, tots = conn.sql(<<~"").to_a[0]
+    select
+      count(distinct(ifnull("#{name}",0))),
+      count(ifnull("#{name}",0))
+    from
+      "#{tabl}"
+    #{filt}
+
+  display(name, data, show, uniq, tots)
+end

data/slyce.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = "slyce"
-  s.version     = `grep '^VERSION' bin/slyce | cut -f 2 -d '"'`
+  s.version     = `grep -m 1 '^\s*@version' bin/slyce | cut -f 2 -d '"'`
   s.author      = "Steve Shreeve"
   s.email       = "steve.shreeve@gmail.com"
   s.summary     =
@@ -11,6 +11,7 @@ Gem::Specification.new do |s|
   s.license     = "MIT"
   s.files       = `git ls-files`.split("\n") - %w[.gitignore]
   s.executables = `cd bin && git ls-files .`.split("\n")
+  s.add_runtime_dependency "any_ascii", "~> 0.3.2"
   s.add_runtime_dependency "extralite-bundle", "~> 1.25"
   s.add_runtime_dependency "mysql2", "~> 0.5"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: slyce
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.2.0
 platform: ruby
 authors:
 - Steve Shreeve
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-13 00:00:00.000000000 Z
+date: 2023-03-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: any_ascii
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   name: extralite-bundle
   requirement: !ruby/object:Gem::Requirement
@@ -43,6 +57,7 @@ email: steve.shreeve@gmail.com
 executables:
 - slyce
 - slyce3
+- slyced
 extensions: []
 extra_rdoc_files: []
 files:
@@ -51,6 +66,7 @@ files:
 - README.md
 - bin/slyce
 - bin/slyce3
+- bin/slyced
 - slyce.gemspec
 homepage: https://github.com/shreeve/slyce
 licenses: