abstract-tables 1.0.0

data/README.textile

@@ -0,0 +1,76 @@
+h1. Abstract Table Library
+
+In the spirit of the standard Unix utilites (like cat, grep, cut, etc.), this library creates an abstraction over tabular data.  The library implements a series of drivers for sources with different encodings that act like sequences.
+
+
+h1. Command Line Utilities
+
+h2. atcat
+
+<pre>
+# dump a postgres table to the console (default is tab delimited)
+atcat dbi://pg/localhost/database_name/table_name
+
+# export a table to a tab file:
+atcat dbi://pg/localhost/database_name/table_name tab://table_name.tab
+
+# you can omit the schema for files, the schema will be guessed based on the file extension
+atcat dbi://pg/localhost/database_name/table_name table_name.tab
+
+# dump to csv
+atcat dbi://pg/localhost/database_name/table_name csv://table_name.csv
+# you can omit the scehma
+atcat dbi://pg/localhost/database_name/table_name table_name.csv
+
+# convert from csv to tab
+atcat csv://some-file.csv tab://some-file.tab
+
+atcat some-file.csv some-file.tab
+
+# convert form csv to pipe
+atcat csv://some-file.csv 'tab://some-file.tab?col_sep=|'
+</pre>
+
+h2. rec-view
+
+<pre>
+rec-view tab://some-file.tab | less
+rec-view csv://some-file.csv | less
+rec-view dbi://Pg/some-file.csv | less
+</pre>
+
+h2. atcat
+
+Limitations: only implemented 'driver' is dbi.  Only supported output port is stdout as tab delimited.
+
+<pre>
+atcat dbi://user:pass@Pg/localhost/db_name/table_name | rec-view | less
+</pre>
+
+h1. Suported Drivers
+
+h2. tab
+
+Native Ruby for now.
+
+h3. col_sep
+
+Defaults to a tab character.
+
+h2. csv
+
+Via the "FasterCSV":http://fastercsv.rubyforge.org/ ruby gem.
+
+h3. col_sep
+
+Override the default ',' column seperator.
+
+h3. quote_char
+
+Override the default quote_char (").
+
+h1. License
+
+h1. Authors
+
+Kyle Burton <kyle.burton@gmail.com>

data/abstract-tables.gemspec

@@ -0,0 +1,45 @@
+require 'rubygems'
+
+SPEC = Gem::Specification.new do |s|
+  s.name = "abstract-tables"
+  s.version = "1.0.0"
+  s.date = '2011-02-09'
+  s.authors = ["Kyle Burton"]
+  s.email = "kyle.burton@gmail.com"
+  s.platform = Gem::Platform::RUBY
+  s.description = <<DESC
+The best I could come up with was to just show you: 
+
+  $ atcat dbi://pg/localhost/database_name/table_name csv://table_name.csv
+
+That exports a table from Postgres into a comma separated value file.  You can 
+read from or write to: tab, csv, dbi, etc.  You can opaquely treat any of those
+'table of records' based sources as an opaque URI.  Want to read more?
+
+   https://github.com/kyleburton/abstract-tables
+
+DESC
+  s.summary = "Table Abstraction as a URI : Record Streams, Filters, ETL Ginsu"
+  s.homepage = "http://github.com/kyleburton/abstract-tables"
+  s.files = %w[
+    abstract-tables-1.0.0.gem
+    abstract-tables.gemspec
+    bin/atcat
+    bin/atview
+    introducing-abtab/README.textile
+    lib/abtab/driver.rb
+    lib/abtab/drivers/csv_driver.rb
+    lib/abtab/drivers/dbi_driver.rb
+    lib/abtab/drivers/tab_driver.rb
+    lib/abtab.rb
+    README.textile
+    test/fixtures/files/file1.csv
+    test/fixtures/files/file1.tab
+  ]
+  puts "all files: #{s.files.inspect}"
+  s.executables = %w[atcat atview]
+  s.require_paths = %w[lib bin]
+  s.extra_rdoc_files = %w[README.textile] #  LICENSE]
+  s.add_runtime_dependency('dbi', [">= 0.4.5"])
+  s.has_rdoc = false
+end

data/bin/atcat

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'abtab'
+
+# TODO: if stdin is a pipe, open it as tab:///dev/stdin
+input_uri = nil
+if File.pipe?('/dev/stdin')
+  input_uri = 'tab:///dev/stdin'
+else
+  input_uri = ARGV.shift or raise "You must speicfy a URI to cat"
+end
+
+inp = Abtab.read_handle input_uri
+
+outp = nil
+if ARGV.empty?
+  outp = Abtab.write_handle 'tab:///dev/stdout'
+else
+  outp = Abtab.write_handle ARGV.shift
+end
+
+outp.set_columns inp.columns
+
+outp.import inp
+

data/bin/atview

@@ -0,0 +1,136 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'abtab'
+
+#
+# Pipe mysql through this:
+#
+#    mysql -uuser database_name -e 'select * from some_table' | rec-view.rb
+#
+# and it will produce a 'portrait' view of the records
+#
+# Record[1]
+# [  1] ID   : 1
+# [  2] FNAME: Kyle
+# [  3] LNAME: Burton
+# [  4] CITY : Philadelphia
+# [  5] ST   : PA
+# [  6] ZIP  : 19101
+# 
+# Record[2]
+# [  1] ID   : 2
+# [  2] FNAME: Alan
+# [  3] LNAME: Barton
+# [  4] CITY : Newardk
+# [  5] ST   : DE
+# [  6] ZIP  : 10817
+#
+
+require 'optparse'
+
+# $options = {:no_hdr => false}
+# OptionParser.new do |opts|
+#   opts.banner = "Usage: #$0 [options]"
+# 
+#   opts.on("-s", "--sort", "Sort the fields before display") do |v|
+#     $options[:sort_fields] = v
+#   end
+# 
+#   opts.on("-f", "--fields FIELDS", "Specify the fields as a comma delimited string") do |v|
+#     $options[:fields] = v
+#   end
+# 
+#   opts.on("-c", "--const N:V,N:V,...", "Specify constant field values, always prefixed, comma separated") do |v|
+#     $options[:constant_fields] = v
+#   end
+# 
+#   opts.on("-r", "--raw", "Raw output, just tab delimited") do |v|
+#     $options[:raw] = v
+#   end
+# 
+#   opts.on("-n", "--no-header", "Suppress printing the header for --raw") do |v|
+#     $options[:no_hdr] = true
+#   end
+# 
+# end.parse!
+# 
+# def constant_field_names
+#   $options[:constant_fields].split(/,/).map { |pair|
+#     pair.split(/:/)[0]
+#   }
+# end
+# 
+# def constant_field_values
+#   $options[:constant_fields].split(/,/).map { |pair|
+#     pair.split(/:/)[1]
+#   }
+# end
+# 
+# fields = []
+# if $options[:fields]
+#   fields = $options[:fields].split(/,/)
+# else
+#   header_line = $stdin.readline
+#   fields = header_line.split(/\t/)
+#   fields[-1].chomp!
+# end
+# 
+# fields.unshift(*constant_field_names) if $options[:constant_fields]
+# 
+# orig_order_map = {}
+# fields.each_with_index do |field,idx|
+#   orig_order_map[field] = idx
+# end
+# sorted_fields = $options[:sort_fields] ? fields.sort : fields
+# 
+# max_width = fields.map {|f| f.size}.max
+# recno = 0
+# 
+# if $options[:raw]
+#   puts fields.join("\t") if !$options[:no_hdr]
+#   $stdin.each do |line|
+#     recno = recno + 1
+#     rec = line.split(/\t/)
+#     rec[-1].chomp!
+#     rec.unshift(*constant_field_values) if $options[:constant_fields]
+#     puts rec.join("\t")
+#   end
+#   exit 0
+# end
+# 
+# $stdin.each do |line|
+#   recno = recno + 1
+#   puts "Record[#{recno}]"
+#   rec = line.split(/\t/)
+#   rec[-1].chomp!
+#   rec.unshift(*constant_field_values) if $options[:constant_fields]
+#   rows = []
+#   sorted_fields.each_with_index do |field,idx|
+#     actual_idx = orig_order_map[field]
+#     rows << sprintf( "[% 3d] %-*s: %s\n", 1+idx, max_width, field, rec[actual_idx] )
+#   end
+#   if $options[:sort_fields]
+#     puts rows.sort
+#   else
+#     puts rows
+#   end
+#   puts ""
+# end 
+
+uri = ARGV.shift or raise "You must speicfy a URI to view"
+inp = Abtab.read_handle uri
+recno = 0
+max_width = inp.columns.map {|f| f.size}.max
+
+while rec = inp.next_record
+  break if rec.nil?
+  recno = recno + 1
+  puts "Record[#{recno}]"
+  rows = []
+  inp.columns.each_with_index do |field,idx|
+    rows << sprintf( "[% 3d] %-*s: %s\n", 1+idx, max_width, field, rec[idx] )
+  end
+
+  puts rows
+  puts ""
+end 

data/introducing-abtab/README.textile

@@ -0,0 +1,50 @@
+I came to Ruby by way of Perl.
+
+  The Perl archipelligo, Wall Island, Chiristiansen, Schwartz, Conway
+
+* gasp *
+
+I started with Perl 4 in order to customize the installation of Windows NT so
+that it could be automated.
+
+This involved heavy use of regular expressions.
+
+  Mom: "Bobby, we need to talk"
+       "What's is Mom?"                  : Bobby
+  Mom: "A freiend's mother said
+        other kids at school
+        have been expeirmenting
+        with the regexes..."
+       "It's no big deal Mom, they're    : Bobby
+        just =~ /(posix|pcre)/ims"
+  Mom: "OMG, you're taking the regex!"
+
+Why regexes?
+
+  Some people, when confronted with a problem, think "I know, I'll use regular
+  expressions." Now they have two problems.  
+
+     -- Jamie Zawinski
+
+Regexes are a key that unlocks data
+
+  Find/Comission a graphic of a key-shaped word 'regex' and the word 'data' shaped
+  as an old style padlock (where the key goes in the front/middle)?
+
+When I worked at Health Market Science, Ben Kennedy and I were asked to help
+out our data delivery department with a client job that was taking at the time
+about 48hrs to process.
+
+During that time we happened upon the idea of a suite of command line tools and
+a library which abstracted away file encodings and how tables of data were to
+be represnted.  
+
+Some clients wanted tab files, some csv, some pipe delimited, some Excel and
+others MS Access databse files.
+
+The other idea that we stumbled upon was to use a URL to represent both the
+location of the table's data and its encoding.  This meant the command line utilities and the libraries coudl be written to outputs, 
+
+  Picture(s):
+    URI: Yuri Gregarin
+    URL: the Ural Mountains

data/lib/abtab.rb

@@ -0,0 +1,57 @@
+require 'abtab/driver'
+
+module Abtab
+  REGISTERED_DRIVERS = {}
+
+  def self.register schema, driver
+    REGISTERED_DRIVERS[schema] = driver
+  end
+
+  def self.open thing
+    if thing !~ /^.+?:\/\//
+      return self.open_file thing
+    end
+
+    viable_drivers = REGISTERED_DRIVERS.keys.select do |schema|
+      thing.start_with? schema
+    end
+
+    if 0 == viable_drivers.size
+      raise "Error: there is no registered driver for url: #{thing}"
+    end
+
+    if 1 == viable_drivers.size
+      REGISTERED_DRIVERS[viable_drivers.first].new thing
+    end
+  end
+
+  def self.read_handle uri
+    driver = self.open uri
+    driver.open_for_reading
+    driver
+  end
+
+  def self.write_handle uri
+    driver = self.open uri
+    driver.open_for_writing
+    driver
+  end
+
+  def self.open_file thing
+    # TODO: put in some more 'magick' -- use some hueristics to figure out the likely format by looking at the first few lines...
+    if thing =~ /.tab$/
+      return self.open "tab://#{thing}"
+    end
+
+    if thing =~ /.csv$/
+      return self.open "csv://#{thing}"
+    end
+
+    # tab delimited is the default
+    return self.open "tab://#{thing}"
+  end
+end
+
+Dir[File.dirname(__FILE__) + '/abtab/drivers/**/*.rb' ].each do |f|
+  require f
+end

data/lib/abtab/driver.rb

@@ -0,0 +1,24 @@
+module Abtab
+  class Driver
+    def import inp
+      while rec = inp.next_record
+        break if rec.nil?
+        write_record rec
+      end
+    end
+
+    def url_parse url, options={}
+      schema, rest = url.split '://', 2
+      path, qs = rest.split '?', 2
+      if qs
+        qs.split(/[;&]/).each do |pair|
+          k,v = pair.split '='
+          k = URI.unescape k
+          v = URI.unescape v
+          options[k] = v
+        end
+      end
+      return schema, path, options
+    end
+  end
+end

data/lib/abtab/drivers/csv_driver.rb

@@ -0,0 +1,68 @@
+require 'fastercsv'
+require 'uri'
+
+class Abtab::Driver::CSVDriver < Abtab::Driver
+  def initialize url
+    options = {
+      "quote_char" => '"',
+      "col_sep"    => ','
+    }
+    @schema, @file, @options = url_parse url, options
+  end
+
+  def open_for_reading
+    if @file == '/dev/stdin'
+      @read_fh = $stdin
+    else
+      if !File.exists? @file
+        raise "Error: can not open for reading, file does not exist: #{@file}"
+      end
+      @read_fh = File.open(@file,'r')
+    end
+
+    header_line = @read_fh.readline
+    header_line.chomp!
+    @columns = FasterCSV.parse(header_line, :quote_char => @options["quote_char"], :col_sep => @options["col_sep"]).first
+  end
+
+  def columns
+    @columns
+  end
+
+  def next_record
+    return nil if @read_fh.eof?
+    line = @read_fh.readline
+    line.chomp!
+    FasterCSV.parse(line, :quote_char => @options["quote_char"], :col_sep => @options["col_sep"]).first
+  end
+
+  def close
+    if @read_fh
+      @read_fh.close
+      @read_fh = nil
+    end
+  end
+
+  def open_for_writing
+    # NB: truncates the output file
+    if @file == '/dev/stdout'
+      @write_fn = $stdout
+    else
+      @write_fh = File.open(@file,'w')
+    end
+    set_columns(@columns) if @columns && !@columns.empty
+  end
+
+  def write_record rec
+    line = rec.to_csv(:quote_char => @options["quote_char"], :col_sep => @options["col_sep"])
+    @write_fh.puts line
+  end
+
+  def set_columns cols
+    @columns = cols
+    write_record @columns
+  end
+
+end
+
+Abtab.register 'csv', Abtab::Driver::CSVDriver

data/lib/abtab/drivers/dbi_driver.rb

@@ -0,0 +1,149 @@
+require 'dbi'
+require 'yaml'
+
+class Abtab::Driver::DbiDriver < Abtab::Driver
+  CREDENTIALS_RC_FILE = "#{ENV['HOME']}/.abtab.dbrc"
+
+  DB_MODULES = {
+    'Pg'      => { :require => 'Pg', :name => 'Pg' },
+    'pg'      => { :require => 'Pg', :name => 'Pg' },
+    # TODO: test these
+    #'mysql'   => 'mysql',
+    #'Mysql'   => 'mysql',
+    #'sqlite'  => 'sqlite3',
+    #'SQLite'  => 'sqlite3',
+    #'sqlite3' => 'sqlite3',
+    #'SQLite3' => 'sqlite3',
+  }
+
+  def lookup_rc_default *keys
+    #puts "LOOKUP: #{keys.inspect}"
+    m = @rc_defaults
+    keys.each do |k|
+      #puts "  k=#{k} m=#{m.inspect}"
+      m = m[k] if m
+    end
+    m
+  end
+
+  def initialize url
+    @options = {
+      :dbi_driver => nil,
+      :host       => nil,
+      :user       => nil,
+      :pass       => nil,
+      :database   => nil,
+      :table      => nil,
+    }
+
+    @rc_defaults = {}
+    if File.exist? CREDENTIALS_RC_FILE
+      @rc_defaults.merge!(YAML.load_file(CREDENTIALS_RC_FILE))
+    end
+
+    @url = url
+    schema, rest = url.split '://', 2
+
+    # expect 'driver' name, eg: pg
+    driver, host, database_name, table = rest.split '/', 4
+
+    if driver =~ /@/
+      user_pass, driver = driver.split '@', 2
+      user, pass = nil, nil
+      if user_pass =~ /:/
+        user, pass = user_pass.split ':', 2
+        @options[:user] = user
+        @options[:pass] = pass
+      end
+      @options[:dbi_driver] = driver
+    end
+
+    @options[:dbi_driver] = driver
+    @options[:database]   = database_name
+    @options[:host]       = host
+    @options[:table]      = table
+
+    @options[:user] ||= lookup_rc_default @options[:host], @options[:database], "user"
+    @options[:pass] ||= lookup_rc_default @options[:host], @options[:database], "pass"
+
+    #puts "OPTIONS: #{@options.inspect}"
+
+    require DB_MODULES[@options[:dbi_driver]][:require]
+    driver_name = DB_MODULES[@options[:dbi_driver]][:name]
+
+    @conn = DBI.connect("DBI:#{driver_name}:database=#{@options[:database]};host=#{@options[:host]}",@options[:user],@options[:pass]);
+
+  end
+
+  def open_for_reading
+    if @options[:table]
+      @columns = get_col_names @options[:table]
+      @statement_handle = @conn.prepare "select * from #{@options[:table]}"
+      @statement_handle.execute
+    else
+      @columns = ['NAME','ROWS']
+      @rows = @conn.tables
+    end
+  end
+
+  # TODO: implement insertion/append and truncation
+#  def open_for_writing
+#    close_statement_handle
+#    @columns = get_col_names @options[:table]
+#    sql_stmt = sprintf("INSERT INTO #{@options[:table]} VALUES (%s)", get_col_names.map {'?'}.join(","))
+#    puts "sql_stmt: #{sql_stmt}"
+#    @statement_handle = @conn.prepare sql_stmt
+#  end
+
+  def columns
+    @columns
+  end
+
+  def get_col_names table
+    sth = @conn.prepare "SELECT * FROM #{table} where 1=0"
+    sth.execute
+    col_names = sth.column_names
+    sth.finish
+    col_names
+  end
+
+  def next_record
+    return @statement_handle.fetch if @statement_handle
+    if @rows && !@rows.empty?
+      table = @rows.shift
+      sth = @conn.prepare("SELECT COUNT('x') from #{table}")
+      sth.execute
+      count = sth.fetch_array.first
+      sth.finish
+      return [table,count]
+    end
+    return nil
+  end
+
+  def rewind
+    close_connection
+    open_for_reading
+  end
+
+  def close 
+    close_statement_handle
+    close_connection
+  end
+
+  def close_connection
+    if @conn
+      @conn.close
+      @conn = nil
+    end
+  end
+
+  def close_statement_handle
+    if @statement_handle
+      @statement_handle.finish
+      @statement_handle = nil
+    end
+  end
+
+end
+
+Abtab.register 'dbi', Abtab::Driver::DbiDriver

data/lib/abtab/drivers/tab_driver.rb

@@ -0,0 +1,76 @@
+
+class Abtab::Driver::TabDriver < Abtab::Driver
+  def initialize url
+    @options = {}
+    @options["col_sep"] = "\t"
+    @schema, @file, @options = url_parse url, @options
+  end
+
+  def open_for_reading
+    if !File.exists? @file
+      raise "Error: can not open for reading, file does not exist: #{@file}"
+    end
+
+    @read_fh = File.open(@file,'r')
+    header_line = @read_fh.readline
+    @columns = parse_line header_line
+  end
+
+  def parse_line l
+    l.chomp!
+    r = l.split(@options["col_sep"]).map do |f|
+      f.gsub! "\\t", "\t"
+      f.gsub! "\\n", "\n"
+      f.gsub! "\\r", "\r"
+      f
+    end
+    r[0].chomp!
+    r
+  end
+
+  def format_rec r
+    r.map do |f|
+      f.gsub! "\t", "\\t"
+      f.gsub! "\n", "\\n"
+      f.gsub! "\r", "\\r"
+      f
+    end.join(@options["col_sep"])
+  end
+
+  def columns
+    @columns
+  end
+
+  def next_record
+    return nil if @read_fh.eof?
+    parse_line @read_fh.readline
+  end
+
+  def close
+    if @read_fh
+      @read_fh.close
+      @read_fh = nil
+    end
+    if @write_fh
+      @write_fh.close
+      @write_fh = nil
+    end
+  end
+
+  def open_for_writing 
+    @write_fh = File.open(@file, 'w')
+    set_columns(@columns) if @columns && !@columns.empty
+  end
+
+  def write_record rec
+    @write_fh.puts format_rec(rec)
+  end
+
+  def set_columns cols
+    @columns = cols
+    write_record @columns
+    
+  end
+end
+
+Abtab.register 'tab', Abtab::Driver::TabDriver

data/test/fixtures/files/file1.csv

@@ -0,0 +1,3 @@
+col1,col2,col3
+this,that,other
+some,more,stuf

data/test/fixtures/files/file1.tab

@@ -0,0 +1,3 @@
+col1	col2	col3
+this	that	other
+some	more	stuf

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification 
+name: abstract-tables
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Kyle Burton
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-09 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: dbi
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 0
+        - 4
+        - 5
+        version: 0.4.5
+  type: :runtime
+  version_requirements: *id001
+description: |+
+  The best I could come up with was to just show you: 
+  
+    $ atcat dbi://pg/localhost/database_name/table_name csv://table_name.csv
+  
+  That exports a table from Postgres into a comma separated value file.  You can 
+  read from or write to: tab, csv, dbi, etc.  You can opaquely treat any of those
+  'table of records' based sources as an opaque URI.  Want to read more?
+  
+     https://github.com/kyleburton/abstract-tables
+  
+email: kyle.burton@gmail.com
+executables: 
+- atcat
+- atview
+extensions: []
+
+extra_rdoc_files: 
+- README.textile
+files: 
+- abstract-tables-1.0.0.gem
+- abstract-tables.gemspec
+- bin/atcat
+- bin/atview
+- introducing-abtab/README.textile
+- lib/abtab/driver.rb
+- lib/abtab/drivers/csv_driver.rb
+- lib/abtab/drivers/dbi_driver.rb
+- lib/abtab/drivers/tab_driver.rb
+- lib/abtab.rb
+- README.textile
+- test/fixtures/files/file1.csv
+- test/fixtures/files/file1.tab
+has_rdoc: true
+homepage: http://github.com/kyleburton/abstract-tables
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- bin
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: "Table Abstraction as a URI : Record Streams, Filters, ETL Ginsu"
+test_files: []
+