cetusql 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 40c861766945ebcd5aeb412cff9788e0cf0a6292
+  data.tar.gz: 960a09544ab79c76a9718972240b5b470e2e6d91
+SHA512:
+  metadata.gz: 241637ec8ca55d51a70060827a87694550ba44d9e48a5397a807af399e20b9f264a77bbd770f77aceced890ff2f6dfe6994a3ebc404a19cc7b19a03dac48d95b
+  data.tar.gz: c56e7c267813b5a71578dcb7dd16a164cf6825cae3b6253199bc574036de38c1b8959d19eb84cbedcffb053179be5f04b1286ddb1e446347e9f74365b8580063

data/.gitignore

@@ -0,0 +1,17 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.tgz
+*.bak
+t.t
+*.tmp
+*.t
+*.sqlite
+*.db
+

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cetusql.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 rkumar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# Cetusql
+
+Command-line based sqlite3 database explorer. The idea is to minimize typing of basic queries/tablenames and column names by allowing for selection of the same.
+
+
+Some basic features:
+
+- Lists databases in current folder and allows menu for selection.
+- selection of tablenames from list, and of columns names and creates formatted output to a local temp file which 
+is then displayed.
+- bookmark/save common SQL queries and recall later.
+
+Uses sqlite3 gem, readline for editing. In the case of columns, uses `fzf` for selection.
+
+Also, uses 'term-table.rb' for formatting of output. This needs to be placed in your path as an executable.
+
+You may replace that with your own formatter such as `csvlook` or `column -t`. However, these two crash on non-ascii characters on a Mac, so I've written a ruby replacement.
+You can forgo it entirely and just have comma or tab separated output sent to a file.
+
+## Installation
+
+    $ gem install cetusql
+    $ brew install fzf
+    $ brew install readline
+
+## Usage
+
+run `cetusql` on the command line.
+
+If no database name is passed, will prompt for one from *.db and *.sqlite files in current directory.
+After selection, one may select one or more tables from a list.
+Then one may select one ore more columns from a list.
+Edit the SQL statement if need be, and press ENTER.
+The output is displayed in vim using a temp file.
+
+Use the menu to change table or database file, or set other options, save last SQL query, select from favorite queries, etc.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rkumar/cetusql.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "cetusql"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/cetusql.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+#require 'cetusql/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "cetusql"
+  spec.version       = "0.0.1"
+  spec.authors       = ["Rahul Kumar"]
+  spec.email         = ["sentinel1879@gmail.com"]
+
+  spec.summary       = %q{command line based sqlite db navigator}
+  spec.description   = %q{command line based sqlite db navigator. ruby 1.9.3 .. 2.4.}
+  spec.homepage      = "https://github.com/rkumar/cetusql"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    #spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  else
+    #raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  # http://bundler.io/blog/2015/03/20/moving-bins-to-exe.html
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency 'sqlite3', '~> 1.3', '>= 1.3.11'
+end

data/exe/cetussql

@@ -0,0 +1,975 @@
+#!/usr/bin/env ruby
+# ----------------------------------------------------------------------------- #
+#         File: cetusql
+#  Description: Fast database navigator for sqlite
+#       Author: rkumar
+#         Date: 2013-02-17 - 17:48
+#      License: MIT
+#  Last update: 2017-04-04 17:09
+# ----------------------------------------------------------------------------- #
+#  cetussql  Copyright (C) 2012-2017 rahul kumar
+#  == TODO
+#  - when in sql menu keep there, the menu of SQL is visible, it looks like we 
+#    are still there.
+#  + toggle_menu - formatting, columns headers, no-history, no-indexinfo etc
+#  - try to get table name or query name in temp file name
+#  - if in db_menu key is unknown then try global menu and remove some options from other menus
+#    - reduce db_menu some options useless unless sql fired. put under SQL menu "s"
+#  - store history in another file, per database, not one file, getting too large.
+#  - the menus and hotkeys suck , need to be fixed.
+#  - define joins, so they can be reused
+#  + view_schema
+#  + make_query
+#  + menu keeps repeating don't show, can press ENTER or ` to view it
+#  - 
+#  + order by is stored, so picked up against wrong database
+#  + if headers requested then we have to add them.
+#   last_sql must be for current database not just any. save under this database and retrieve under this database
+#   - up arrow on table menu should call edit_last_sql
+#   ? don't redraw the menu. the loop is outside the menu. keep it inside menu() so that it is not redrawn
+#    but exits upon q. what if user does not want that ?
+#
+#   options : unformatted with tabs output - huge outputs take time
+#      STDOUT or file. specify output to a file
+#      header off and on
+#
+#
+#  == END TODO
+require 'readline'
+require 'io/wait' # may not need this
+# http://www.ruby-doc.org/stdlib-1.9.3/libdoc/shellwords/rdoc/Shellwords.html
+require 'shellwords'
+require 'yaml'
+#require 'fileutils'
+# in ~/work/projects/common/
+require 'cetusql/cli_utils'
+require 'cetusql/cli_sqlite'
+# -- requires 1.9.3 or higher for io/wait
+# -- cannot do with Highline since we need a timeout on wait, not sure if HL can do that
+
+VERSION="0.1.0"
+O_CONFIG=true
+CONFIG_FILE=File.expand_path("~/.cetusqlinfo")
+# Using this to store data that has to be used in actions which do not take parameters or return values
+#  when called from a key
+$options = {}
+$options[:headers] = true
+$options[:output_to] = nil
+$options[:formatting] = true
+
+$bindings = {}
+$bindings[:db]  = {
+  :t => :select_table,
+  :s => :sql_menu,
+  :m => :multi_select_table,
+  :l => :list_tables,
+  :q => :quit
+}
+$bindings[:table] = {
+  :v => :view_menu,
+  "4".to_sym => :select_columns,
+  "5".to_sym => :select_orderby,
+  "6".to_sym => :select_where,
+  "R".to_sym => :run_query,
+  "0".to_sym => :make_query, 
+  :s => :save_sql,
+  :x => :new_query, # DDL no result enter a query and execute
+  :h => :history_menu,  # queries for this table
+  :e => :edit_last_sql,
+  :O => :output_to,
+  :d => :change_database,
+  :j => :define_join,
+  "UP" => :edit_last_sql,
+  :q => :quit
+}
+$old_bindings = {
+  "`"   => "main_menu",
+  "="   => "toggle_menu",
+  "!"   => "command_mode",
+  "@"   => "selection_mode_toggle",
+  "M-a" => "select_all",
+  "M-A" => "unselect_all",
+  ","   => "goto_parent_dir",
+  "+"   => "goto_dir",
+  "."   => "pop_dir",
+  ":"   => "subcommand",
+  "'"   => "goto_bookmark",
+  "/"   => "enter_regex",
+  "M-p"   => "prev_page",
+  "M-n"   => "next_page",
+  "SPACE"   => "next_page",
+  "M-f"   => "select_visited_files",
+  "M-d"   => "select_used_dirs",
+  "M-b"   => "select_bookmarks",
+  "M-m"   => "create_bookmark",
+  "M-M"   => "show_marks",
+  "C-c"   => "escape",
+  "ESCAPE"   => "escape",
+  "TAB"   => "views",
+  "C-i"   => "views",
+  "?"   => "dirtree",
+  "ENTER"   => "select_current",
+  "D"   => "delete_file",
+  "M"   => "file_actions most",
+  "Q"   => "quit_command",
+  "RIGHT"   => "column_next",
+  "LEFT"   => "column_next 1",
+  "C-x"   => "file_actions",
+  "M--"   => "columns_incdec -1",
+  "M-+"   => "columns_incdec 1",
+  "S"     =>  "command_file list y ls -lh",
+  "L"     =>  "command_file Page n less",
+  "C-d"   =>  "cursor_scroll_dn",
+  "C-b"   =>  "cursor_scroll_up",
+  "UP"   =>  "cursor_up",
+  "DOWN"   =>  "cursor_dn",
+  "C-SPACE" => "visual_mode_toggle",
+
+  "M-?"   => "print_help",
+  "F1"   => "print_help",
+  "F2"   => "child_dirs",
+  "F3"   => "dirtree",
+  "F4"   => "tree",
+  "S-F1"   => "dirtree",
+  "S-F2"   => "tree"
+
+}
+
+
+$mode = nil
+$glines=%x(tput lines).to_i
+$gcols=%x(tput cols).to_i
+$grows = $glines - 3
+$pagesize = 60
+$gviscols = 3
+$pagesize = $grows * $gviscols
+MSCROLL = 10
+$quitting = false
+$modified = $writing = false
+
+#$help = "#{BOLD}M-?#{BOLD_OFF} Help   #{BOLD}`#{BOLD_OFF} Menu   #{BOLD}!#{BOLD_OFF} Command   #{BOLD}=#{BOLD_OFF} Toggle    #{BOLD}q#{BOLD_OFF} Quit "
+$help = "#{BOLD}`#{BOLD_OFF} Menu   #{BOLD}=#{BOLD_OFF} Toggle    #{BOLD}q#{BOLD_OFF} Quit "
+
+db = nil
+
+## main loop which calls all other programs
+def run()
+  ctr=0
+  filename, db, tables = change_database ARGV[0]
+  #filename = ARGV[0] || getdbname
+  #if filename
+    #db = Database.new filename
+    #tables = db.tables
+  #end
+  exit unless db
+  $mode = :db
+  # TODO populate readline with earlier SQLS for this database
+  prompt = ""
+  ch, text = db_menu
+  process_number ch
+  
+
+  while true
+    i = 0
+    # title
+    filename = $g_data[:filename]
+    db = $g_data[:db]
+    #print "#{GREEN}#{$help}  #{BLUE}cetusql #{VERSION} :#{$mode}#{CLEAR}\n"
+    #t = "#{$title}"
+    #t = t[t.size-$gcols..-1] if t.size >= $gcols
+    #print "#{BOLD}#{t}#{CLEAR}\n"
+    # another query , change table, save sql, change db, select multiple tables
+    # schema, count, sample
+    # ZZZ XXX
+    #print "\r#{_mm}#{$patt} >"
+    ch = text = nil
+    #print "\r:#{$mode} >"
+    print "\r:%-5s >" % [$mode]
+    ch = get_char
+    print ch
+    # TODO
+    # depending on mode, we display menu if tilde pressed, or we process based on tilde
+    ch, text = process_key ch
+    puts ch if $opt_debug
+    if ch == "`" or ch == "ENTER"
+      if $mode == :table
+        display_menu "Menu for #{$mode} #{current_tablename}", $bindings[$mode]
+      else
+        display_menu "Menu for #{$mode} #{current_dbname}", $bindings[$mode]
+      end
+    elsif ch == '='
+      toggle_menu
+    end
+    if ch == "q" or ch == "ESCAPE" or ch == "C-c"
+      if $mode == :table
+        $mode = :db
+        $g_data[:current_tablename] = nil
+      else
+        break
+      end
+    end
+    #puts
+    #break if ch == "q" or ch == "ESCAPE" or ch == "C-c"
+  end
+  puts "bye"
+  $writing = true
+  config_write if $writing
+end
+def process_key ch
+  # get correct action map
+  h = $bindings[$mode] || $bindings[:db]
+  binding = h[ch]
+  binding = h[ch.to_sym] unless binding
+  if binding
+    if respond_to?(binding, true)
+      # 2017-03-19 - we can't send return values from a method ??
+      send(binding)
+    end
+  else
+    if $mode == :db
+      process_number ch
+    end
+  end
+  return ch, binding
+end
+def config_write
+  $g_data[:db] = nil
+  $g_data[:tables] = nil
+  writeYML $g_data, CONFIG_FILE
+end
+def config_read
+  if File.exist? CONFIG_FILE
+    $g_data = loadYML(CONFIG_FILE)
+  else
+    $g_data = {}
+  end
+end
+
+def loadYML( filename)
+  hash = YAML::load( File.open( filename ) )
+  #puts hash.keys.size
+  return hash
+end
+
+require 'fileutils' # for mkdir_p
+def writeYML obj, filename
+  dir = File.dirname filename
+  unless File.exist? dir
+    FileUtils::mkdir_p dir
+  end
+  File.open(filename, 'w') {|f| f.write obj.to_yaml }
+  #$stderr.puts color("==> written to #{filename}", "green", "bold") unless $opt_quiet
+  $stderr.puts("==> written to #{filename}") unless $opt_quiet
+end
+
+
+def sql_menu
+  # check if there is an sql that has been fired, then save_sql and edit_last
+  # if table selected then select columns, order_by where etc
+  h = {
+    :s => :save_sql,
+    :h => :sql_history,  # databases accessed, tables accessed, sqls issued
+    :e => :edit_last_sql,
+    :O => :output_to,
+    :F => :formatting_toggle,
+    :q => :quit
+  }
+  menu "SQL Menu for #{current_dbname}", h
+end
+def view_menu
+  h = {
+  :a => :view_all_rows,
+  :s => :view_sample,
+  :r => :view_recent,
+  :c => :view_schema, 
+  :q => :quit
+  }
+  menu "View Menu for #{current_tablename}", h
+end
+def db_menu
+  h = {
+    :t => :select_table,
+    :m => :multi_select_table,
+    :l => :list_tables,
+    :s => :sql_menu,
+    #:d => :change_database,
+    "UP" => :edit_last_sql,
+    :q => :quit
+  }
+  menu "DB Menu for #{current_dbname}", h
+end
+def toggle_menu
+  h = { :f => :formatting_toggle, :h => :toggle_headers, :i => :toggle_index_info , "H" => :toggle_history, 
+  :p => :toggle_pager_mode}
+  ch, menu_text = menu "Toggle Menu", h
+  case menu_text
+  when :toggle_headers
+     $options[:headers] = !$options[:headers]
+     pgreen "headers is #{$options[:headers]}"
+  when :toggle_index_info
+     # should index info be printed along with columsn, may take time if many tables
+     $options[:index_info] = !$options[:index_info]
+     pgreen "index_info calculation is #{$options[:index_info]}"
+  when :toggle_history
+     $options[:history_save] = !$options[:history_save]
+     pgreen "Saving SQL history is #{$options[:history_save]}"
+  when :toggle_pager_mode
+    $editor_mode = !$editor_mode
+    if $editor_mode
+      $default_command = nil
+    else
+      $default_command = ENV['MANPAGER'] || ENV['PAGER']
+    end
+  end
+end
+def process_number ch
+  # this is done in :db mode, to quick select a table, not in table mode since numbers are used
+  #  if number is < 10 then see if there's a table and select that table
+  puts ch if $opt_debug
+  chi = ch.to_i
+  if chi > 0 and chi < 10
+    tables = fetch(:tables)
+    tablename = tables[chi - 1]
+    if tablename
+      pgreen "Selected  #{tablename}  "
+      select_table(tablename)
+      display_menu "Menu for #{tablename}", $bindings[$mode]
+    end
+  end
+end
+def table_menu tablename=nil
+  t = tablename || current_tablename()
+  h = {
+    "2".to_sym => :view_all_rows,
+    "3".to_sym => :view_sample,
+    "9".to_sym => :view_recent,
+    "4".to_sym => :select_columns,
+    "5".to_sym => :select_orderby,
+    "6".to_sym => :select_where,
+    "R".to_sym => :run_query,
+    "0".to_sym => :make_query, 
+    :s => :save_sql,
+    :x => :new_query, # DDL no result enter a query and execute
+    :h => :history_menu,  # queries for this table
+    :c => :view_schema, 
+    :e => :edit_last_sql,
+    :O => :output_to,
+    :d => :change_database,
+    :j => :define_join,
+    "UP" => :edit_last_sql,
+    :q => :quit
+  }
+  menu "Table Menu for #{t}", h
+end
+
+# TODO
+# 2017-03-27 - construct a query and run it.
+# actually it should give a submenu
+def make_query 
+  # 1. select table if nil
+  # 2 select columns for display
+  # 3. select order
+  # 4. select where
+  # 5. select LIMIT
+  # 6. edit and run
+  tablename = current_tablename()
+  unless tablename
+    tablename = select_table
+  end
+  return unless tablename
+  puts "Select columns"
+  sel_columns = select_columns
+  puts "Select where condition"
+  where_str   = select_where
+  puts "Select order by"
+  orderby_cols = select_orderby
+  run_query
+end
+def new_query
+  db = current_db()
+  tablename = current_tablename()
+  columns = db.columns(tablename)
+  #sql = "SELECT #{columns.join(",")} FROM #{tablename} ;"
+  sql = "SELECT\n#{columns.join(",\n")} \nFROM #{tablename} \nLIMIT 1000\n ;"
+  puts "created sql" if $opt_debug
+  edit_execute_in_editor sql
+end
+def edit_execute_in_editor sql
+  require 'tempfile'
+  tmpfile = Tempfile.new('SQL.XXXXXX')
+  filename = tmpfile.path
+  filename = Shellwords.escape(filename)
+  tmpfile.write(sql)
+  puts "written to #{filename}" if $opt_debug
+  tmpfile.close
+  mtime = File.mtime(filename)
+  puts mtime if $opt_debug
+  command = nil
+  #system "vim #{filename}" and (command = File.open(filename).readlines.join(" ") )
+  system "vim #{filename}" 
+  mtime2 = File.mtime(filename)
+  puts mtime2 if $opt_debug
+  # we are comparing modification times of the file to see if user quit or saved.
+  # vim returns a zero in both cases, unless user quits using :cq
+  return if mtime2 == mtime
+  #puts "not returnig"
+  command = File.open(filename).readlines.join(" ") 
+  if command
+    command = command.gsub("\n", " ")
+    #puts "got : #{command}" if command
+    puts "..."
+    view_sql command
+  end
+  set_last_sql command
+end
+
+def view_schema
+  tablename = current_tablename
+  puts
+  pbold "Schema for #{tablename}"
+  list_metadata tablename
+  indexes_for_table tablename
+  puts
+end
+
+def change_database filename=nil
+  # add to g_data as visited_dbs
+  filename = select_database_name unless filename
+  if filename
+    db = Database.new filename
+    tables = db.tables
+    $g_data[:filename] = filename
+    $g_data[:db] = db
+    $mode = :db
+    $g_data[:current_tablename] = nil
+
+    #$g_data[:tables] = tables
+    store(:tables, tables)
+    $g_data[:visited_dbs] ||= []
+    # TODO should be unique. remove if exists, then add
+    $g_data[:visited_dbs].delete(filename)
+    $g_data[:visited_dbs] << filename
+    $g_data[:last_sql] = nil
+    clear_screen
+    puts "#{BLUE}Using #{current_dbname}#{CLEAR}"
+    list_tables
+    list_indexes
+  end
+  return filename, db, tables
+end
+# get database names
+def select_database_name
+  # Add recent ones, but remove directory portion if they belong to current dir
+  choices = Dir['*.db','*.sqlite','*.sqlite3'] | $g_data[:visited_dbs].map {|e| e.sub(Dir.pwd + "/","") }
+  if choices
+    if choices.size == 1
+      return File.expand_path(choices.first)
+    else
+      # select_from is in cli_utils.rb
+      #db = select_from "Select database", choices
+      #db = ctrlp choices
+      db = choose choices
+      return File.expand_path(db) if db
+    end
+  end 
+  return nil
+end
+#alias :select_database_name :getdbname
+def multi_select_table
+  db = current_db
+  sel_tables = multi_select "select tables ", db.tables
+  columns = []
+  sel_tables.each do |tablename|
+    c = db.columns tablename
+    c.each {|e| columns << "#{tablename}.#{e}" }
+  end
+  sel_columns = multi_select "select columns ", columns
+  #puts sel_columns.size
+  #puts sel_columns.class
+  sel_columns ||= ['*']
+  sql = "SELECT #{sel_columns.join(",")} FROM #{sel_tables.join(',')} \nWHERE\n LIMIT 1000\n ;"
+  puts sql
+  edit_execute_sql sql
+  # TODO edit it and view result
+  # TODO join based on commond fields, we have that code somewhere
+
+end
+def define_join
+  db = current_db
+  tablename = current_tablename()
+
+  sel_table = single_select "select lookup table for join:", db.tables
+  sel_table = sel_table.chomp
+  unless sel_table
+    return
+  end
+  columns = db.columns sel_table
+  sel_columns = multi_select "select columns to join ", columns
+  sel_columns_this = multi_select "select columns from this table ", db.columns( tablename)
+ 
+  str = tablename + "." + sel_columns_this.first + " = " + sel_table + "." + sel_columns.first 
+  puts str
+  ca = sel_columns_this.first
+  cb = sel_columns.first
+  hash_put([:joins, tablename, sel_table, ca, cb])
+   
+  #puts sel_columns.size
+  #puts sel_columns.class
+  # TODO edit it and view result
+  # TODO join based on commond fields, we have that code somewhere
+
+end
+# for current database create hash entries from a heirarchy, like mkdir -p
+def hash_put array
+  h = $g_data[:databases][current_dbname()]
+  array.each do |e|
+    h[e] ||= {}
+    h = h[e]
+  end
+end
+
+# select a table and then puts user into table_menu (that is bad!)
+def select_table tablename=nil
+  db = $g_data[:db]
+  filename = $g_data[:filename]
+  unless tablename
+    tablename = select_from "Select table from:(#{filename})", db.tables
+  end
+  unless tablename
+    if confirm("Do you wish to quit ? ")
+      $quitting = true
+      return false
+    end
+    return
+  end
+  clear_screen
+
+  $g_data[:current_tablename] = tablename
+  list_metadata tablename
+  indexes_for_table tablename
+  joins_for_table tablename
+  $mode = :table
+  return
+end
+# store some values against the current database
+def store key, value
+  $g_data[:databases][current_dbname()] ||= {}
+  $g_data[:databases][current_dbname()][key] = value
+  #$g_data[key] = value
+end
+# retrieve some values against the current database
+def fetch key
+  $g_data[:databases][current_dbname()] ||= {}
+  $g_data[:databases][current_dbname()][key]
+  #$g_data[key]
+end
+def select_columns
+  #puts "inside select_columns"
+  db = current_db()
+  tablename = current_tablename || return
+  columns = db.columns tablename
+  # TODO
+  # display columns with datatypes and indexes on them
+  # count of rows
+  # options for sample, first 100 rows last 100 rows (based on count)
+  # option for selecting column, where, order, RUN, all_cols,
+  sel_columns = multi_select "select column from #{tablename}", columns
+  #puts sel_columns.size
+  #puts sel_columns.class
+  #p sel_columns
+  sel_columns = ['*'] if sel_columns.size == 0
+  # FIXME these should be stored against a specific database and table, otherwise they will cause
+  #  errors
+  store ':selected_columns', sel_columns
+  puts fetch(':selected_columns')
+  return sel_columns
+end
+def select_orderby
+  db = current_db()
+  tablename = current_tablename || exit(11)
+  columns = db.columns tablename
+  sel_columns = multi_select "select order by from #{tablename}", columns
+  sel_columns = nil if sel_columns.size == 0
+  store ':order_by', sel_columns
+  puts fetch(':order_by')
+  return sel_columns
+end
+def select_where
+  db = current_db()
+  tablename = current_tablename || exit(1)
+  columns = db.columns tablename
+  sel_columns = multi_select "select WHERE from #{tablename}", columns
+  sel_columns = nil if sel_columns.size == 0
+  res = nil
+  if sel_columns
+    a = []
+    sel_columns.each do |e|
+      s = input "WHERE #{e} "
+      a << "#{e} #{s}"
+    end
+    res = a.join(" AND ")
+  end
+  store ':where', res
+  puts fetch(':where')
+  return res
+end
+def run_query
+  t =  current_tablename
+  c =  fetch(':selected_columns') || ['*']
+  o =  fetch(':order_by') 
+  w =  fetch(':where') 
+  sql = "SELECT #{c.join(', ')} FROM #{t} "
+  if w
+    #sql += " WHERE #{w.join(', ')} "
+    sql += " WHERE #{w} "
+  end
+  if o
+    sql += " ORDER BY #{o.join(', ')} "
+  end
+  edit_execute_in_editor sql
+  #sql = vared sql, "Edit SQL: "
+  #return if sql.nil? or sql.size == 0
+  #set_last_sql sql
+  #view_sql sql
+end
+def edit_execute_sql sql
+  command = vared sql, "Edit SQL:"
+  if command.nil? or command.size == 0
+    return false
+  end
+  puts "..."
+  view_sql command
+  set_last_sql command
+  return true
+end
+def list_tables
+  db = current_db()
+  tables = current_db.tables
+  puts
+  data = [["#", "Table", "Rows", "Indexed"]]
+  ctr = 1
+  tables.each do |t|
+    #columns, datatypes = db.get_metadata t
+    content = db.get_data "SELECT count(1) from #{t} "
+    rows = content.first.first
+    indexed = '?'
+    if $options[:index_info]
+      indexed = db.indexed_columns(t) || "---"
+    end
+    char = ctr < 10 ? ctr.to_s : ""
+    row = [char, t, rows, indexed]
+    data << row
+    ctr += 1
+    #puts "#{t}  #{rows} "
+  end
+  view_array data
+end
+# I don't need to loop through tables SILLY !
+# FIXME just loop without tables
+def list_indexes
+  db = current_db()
+  data = [["Index", "Table", "Column/s"]]
+  #columns, datatypes = db.get_metadata t
+  sql = %Q{SELECT name, tbl_name, sql FROM sqlite_master WHERE type = "index" ORDER BY tbl_name }
+  content = db.get_data sql
+  return if content.nil? or content == []
+  content.each do |r|
+    if r[-1] != nil
+      m = r[-1].match /\((.*)\)/
+        r[-1] = m[1] if m
+    end
+    row = [ *r ]
+    data << row
+  end
+  view_array data
+end
+def indexes_for_table table
+  db = current_db()
+  data = [["Index", "Table", "Column/s"]]
+  sql = %Q{SELECT name, tbl_name, sql FROM sqlite_master WHERE type = "index" and tbl_name =  "#{table}" }
+  content = db.get_data sql
+  if content.nil? or content == []
+    pred "No indexes for table #{table}"
+    return
+  end
+  title = "Indexes of table: #{table}"
+  puts "-"*title.size
+  pbold title
+  puts "-"*title.size
+  content.each do |r|
+    if r[-1] != nil
+      m = r[-1].match /\((.*)\)/
+        r[-1] = m[1] if m
+    end
+    row = [ *r ]
+    data << row
+  end
+  view_array data
+end
+def joins_for_table tablename
+  arrow = "->"
+  checkmark = "\u2523"
+  checkmark = "\u279C"
+  db = current_db()
+  h = fetch(:joins)
+  return unless h
+  hj = h[tablename]
+  return unless hj
+  #p hj
+  pbold "Joins for table: #{tablename}"
+  hj.each_pair do |k, v|
+    puts "   #{tablename} -> #{k} "
+    v.each_pair do |k1, v1|
+      puts "     #{checkmark} #{tablename}.#{k1} = #{k}.#{v1.keys.first}"
+    end
+  end
+end
+# if too long split and columnate at 10 rows
+def list_metadata table
+  db = current_db()
+  columns, datatypes = db.get_metadata table
+  array = []
+  title = "Columns of table: #{table}"
+  puts "-"*title.size
+  pbold title
+  puts "-"*title.size
+
+    columns.each_with_index do |e, ix|
+    #print "%-20s %s\n" % [e, datatypes[ix] ]
+    array << " %-20s  #{GREEN}%-8s#{CLEAR} " % [e, datatypes[ix] ]
+  end
+  if false
+    array = columnate array, $grows - 7
+    array.each {|line| print line, "\n"  }
+  end
+  print_in_cols array
+  #array.to_table
+
+end
+def config
+  $g_data[:databases][current_dbname()]
+end
+def sql_history
+  dbc = config()
+  hist = dbc[:history]
+  saved = dbc[:saved_sqls]
+  ctr = 1
+  all = []
+  if saved
+    pbold "Saved sqls"
+    saved.each_with_index {|e,i| puts "#{ctr} #{e}"; ctr += 1; }
+    all += saved
+  end
+  if hist
+    pbold "History"
+    hist.each_with_index {|e,i| puts "#{ctr} #{e}"; ctr += 1; }
+    all += hist
+  end
+  print "Select an sql: "
+  choice = $stdin.gets.chomp
+  puts choice
+  return if choice == ""
+  sql = all[choice.to_i - 1]
+  return unless sql
+  sql = vared sql, "Edit sql: "
+  return if sql.nil? or sql.size == 0
+  puts sql
+  view_sql sql
+end
+
+# choose from an array. printed vertically no columns, choose number
+def choose array, title=nil, prompt=': '
+
+  pbold title if title
+  array.each_with_index {|e,i| puts "#{i+1} #{e}" }
+  while true
+    print "> \r (Enter 1 to #{array.size}) "
+    choice = $stdin.gets.chomp
+    if choice == "" or choice == "q"
+      return nil
+    end
+    chi = choice.to_i
+    if chi < 1 or chi > array.size
+      next
+    end
+    break
+  end
+  #puts choice
+  return nil if choice == ""
+  k = array[choice.to_i - 1]
+  return k
+end
+  CSI = "\e["
+def OLDchoose array, title=nil, prompt=': '
+
+  # why is save and resutore not working heee
+  $stdout.write "#{CSI}s"     # save cursor position
+  while true
+    pbold title if title
+    array.each_with_index {|e,i| puts "#{i+1} #{e}" }
+    print prompt
+    choice = $stdin.gets.chomp
+    chi = choice.to_i
+    puts chi
+    if choice == "" or choice == "q"
+      return nil
+    end
+    if chi < 1 or chi > array.size
+      puts "restore"
+      $stdout.write "#{CSI}u"     # restore cursor position
+      next
+    end
+    break
+  end
+    #puts choice
+  return nil if choice == ""
+  k = array[choice.to_i - 1]
+  return k
+end
+  
+def view_sql sql
+  begin
+    view_data current_db, sql, $options
+  rescue => e  
+    puts e.to_s
+    puts e.message  
+    #puts e.backtrace.join("\n")
+  end  
+
+end
+
+
+# send output of following commands to this file
+def output_to filename=nil
+  unless filename
+    filename = input "Enter output filename: "
+    filename = filename.chomp
+  end
+  filename = nil if filename == ""
+  $options[:output_to] = filename
+  return filename
+end
+def formatting_toggle
+  $options[:formatting] = !$options[:formatting]
+  puts "Column formatting is now #{$options[:formatting]}  "
+  pause
+end
+def current_db
+  $g_data[:db]
+end
+def current_dbname
+  $g_data[:filename] || "WARNING NOT SET: :filename"
+end
+def current_tablename
+  $g_data[:current_tablename]
+end
+# save an sql statement like a bookmark
+# Would have liked to have a nickname for it, or title.
+def save_sql sql=nil
+  sql ||= $g_data[:last_sql] 
+  $g_data[:databases] ||= {}
+  $g_data[:databases][current_dbname()] ||= {}
+  $g_data[:databases][current_dbname()][:saved_sqls] ||= []
+  $g_data[:databases][current_dbname()][:saved_sqls].delete sql
+  $g_data[:databases][current_dbname()][:saved_sqls] << sql
+  $g_data[:databases][current_dbname()][:last_sql] = sql
+  pgreen "Saved sql statement"
+  $g_data[:last_sql] = nil
+end
+def set_last_sql sql
+  $g_data[:last_sql] = sql
+  # set history of sqls issued
+  $g_data[:databases] ||= {}
+  $g_data[:databases][current_dbname()] ||= {}
+  $g_data[:databases][current_dbname()][:last_sql] = sql
+  $g_data[:databases][current_dbname()][:history] ||= []
+  $g_data[:databases][current_dbname()][:history].delete sql
+  $g_data[:databases][current_dbname()][:history] << sql
+end
+def edit_last_sql
+  # this can be buggy, if you change database
+  #sql =  $g_data[:last_sql]
+  sql = $g_data[:databases][current_dbname()][:last_sql]
+  #edit_execute_sql sql
+  edit_execute_in_editor sql
+end
+def view_array data
+  filename = tabulate2 data, $options
+  puts "Got  #{filename} " if $opt_verbose
+  File.open(filename).readlines.each  {|line| puts line}
+end
+# display all rows of current table
+def view_all_rows
+  table = current_tablename()
+  #set_last_sql sql
+  sql = "SELECT * from #{table}"
+  view_sql sql
+end
+def view_sample
+  table = current_tablename()
+  #set_last_sql sql
+  sql = "SELECT * from #{table} LIMIT 100"
+  view_sql sql
+end
+# view last 100 inserted rows
+def view_recent
+  table = current_tablename()
+  sql = current_db().sql_recent_rows(table)
+  view_sql sql
+end
+# uses fzf to select an old query
+# fzf messes with quotes, so don't use
+def history_menu
+
+  dbc = config()
+  array = []
+  hist = dbc[:history] || []
+  saved = dbc[:saved_sqls] || []
+  array = saved | hist 
+  unless array
+    perror "No history for this database!"
+    return false
+  end
+  #sql = single_select "Select query: ", array
+  puts "Use up arrow to navigate queries, ^r to search:"
+  command = editline array
+  #command = vared sql, "Edit SQL:"
+  if command.nil? or command.size == 0
+    $quitting = true
+    return false
+  end
+  view_sql command
+  set_last_sql command
+end
+def display_menu title, h
+  puts "inside display_menu with   #{title}, mode = #{$mode}  " if $opt_debug
+    unless h
+      h = $bindings[:db]
+    end
+  return unless h
+  pbold "#{title}"
+  ctr = 0
+  #h.each_pair { |k, v| puts " #{k}: #{v}" }
+  h.each_pair { |k, v| 
+    print " #{k}: %-20s" % [v]
+    if ctr > 1
+      print "\n"
+      ctr = 0
+    else
+      ctr += 1
+    end
+  }
+  print "\n"
+end
+def clear_screen
+  system "clear"
+  t = "#{GREEN}#{$help}  #{BLUE}cetusql #{VERSION}#{CLEAR}"
+  print "#{BOLD}#{t}#{CLEAR}\n"
+end
+  
+
+
+config_read
+$g_data ||= {}
+
+run