tb 0.1

data/lib/tb/qtsv.rb

@@ -0,0 +1,93 @@
+# lib/tb/qtsv.rb - quoted TSV related fetures for table library
+#
+# Copyright (C) 2010 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+class Tb
+
+  # quoted TSV is a variant of TSV (tab separated value)
+  #
+  # All non-empty values are quoted by double-quotes.
+
+  def Tb.load_qtsv(filename, *header_fields, &block)
+    Tb.parse_qtsv(File.read(filename), *header_fields, &block)
+  end
+
+  def Tb.qtsv_stream_input(qtsv)
+    qtsv = qtsv.read unless String === qtsv
+    qtsv = qtsv.dup
+    cells = []
+    verify = ''
+    qtsv.scan(/\G("(.*?)"|)(\t|\n|\r\n|\z)/m) {
+      verify << $&
+      cell = $2
+      sep = $3
+      break if cell.nil? && sep.empty?
+      cells << cell
+      if sep != "\t"
+        yield cells
+        cells = []
+      end
+    }
+    if verify != qtsv
+      if qtsv.start_with?(verify)
+        raise "unexpected scan ('verify' is a prefix of 'qtsv')"
+      end
+      if verify.length != qtsv.length
+        raise "unexpected scan (length differ: orig:#{qtsv.length} verify:#{verify.length})" 
+      end
+      raise "unexpected scan" 
+    end
+    nil
+  end
+
+  def Tb.parse_qtsv(qtsv, *header_fields)
+    aa = []
+    qtsv_stream_input(qtsv) {|ary|
+      aa << ary
+    }
+    aa = yield aa if block_given?
+    if header_fields.empty?
+      aa.shift while aa.first.all? {|elt| elt.nil? || elt == '' }
+      header_fields = aa.shift
+      h = Hash.new(0)
+      header_fields.each_with_index {|f, i|
+        if h.include? f
+          raise ArgumentError, "ambiguous header field: #{f.inspect} (#{h[f]}th and #{i}th)"
+        end
+        h[f] = i
+      }
+    end
+    t = Tb.new(header_fields)
+    aa.each {|ary|
+      h = {}
+      header_fields.each_with_index {|f, i|
+        h[f] = ary[i]
+      }
+      t.insert(h)
+    }
+    t
+  end
+
+end

data/lib/tb/reader.rb

@@ -0,0 +1,213 @@
+# lib/tb/reader.rb - Tb::Reader class
+#
+# Copyright (C) 2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+class Tb
+  def Tb.load_csv(filename, *header_fields, &block)
+    Tb.parse_csv(File.read(filename), *header_fields, &block)
+  end
+
+  def Tb.parse_csv(csv, *header_fields)
+    aa = []
+    csv_stream_input(csv) {|ary|
+      aa << ary
+    }
+    aa = yield aa if block_given?
+    if header_fields.empty?
+      reader = Tb::Reader.new(aa)
+      arys = []
+      reader.each {|ary|
+        arys << ary
+      }
+      header = reader.header
+    else
+      header = header_fields
+      arys = aa
+    end
+    t = Tb.new(header)
+    arys.each {|ary|
+      ary << nil while ary.length < header.length
+      t.insert_values header, ary
+    }
+    t
+  end
+
+  def Tb.load_tsv(filename, *header_fields, &block)
+    Tb.parse_tsv(File.read(filename), *header_fields, &block)
+  end
+
+  def Tb.parse_tsv(tsv, *header_fields)
+    aa = []
+    tsv_stream_input(tsv) {|ary|
+      aa << ary 
+    }
+    aa = yield aa if block_given?
+    if header_fields.empty?
+      reader = Tb::Reader.new(aa)
+      arys = []
+      reader.each {|ary|
+        arys << ary
+      }
+      header = reader.header
+    else
+      header = header_fields
+      arys = aa
+    end
+    t = Tb.new(header)
+    arys.each {|ary|
+      ary << nil while ary.length < header.length
+      t.insert_values header, ary
+    }
+    t
+  end
+end
+
+class Tb::Reader
+  def self.open(filename, opts={})
+    io = nil
+    case filename
+    when /\.csv\z/
+      io = File.open(filename)
+      rawreader = Tb::CSVReader.new(io)
+    when /\.tsv\z/
+      io = File.open(filename)
+      rawreader = Tb::TSVReader.new(io)
+    when /\Acsv:/
+      io = File.open($')
+      rawreader = Tb::CSVReader.new(io)
+    when /\Atsv:/
+      io = File.open($')
+      rawreader = Tb::TSVReader.new(io)
+    else
+      if filename == '-'
+        rawreader = Tb::CSVReader.new(STDIN)
+      else
+        # guess table format?
+        io = File.open(filename)
+        rawreader = Tb::CSVReader.new(io)
+      end
+    end
+    reader = self.new(rawreader, opts)
+    if block_given?
+      begin
+        yield reader
+      ensure
+        reader.close
+      end
+    else
+      reader
+    end
+
+  end
+
+  def initialize(rawreader, opts={})
+    @opt_n = opts[:numeric]
+    @reader = rawreader
+    @fieldset = nil
+  end
+
+  def header
+    return @fieldset.header if @fieldset
+    if @opt_n
+      @fieldset = Tb::FieldSet.new
+    else
+      while ary = @reader.shift
+        if ary.all? {|elt| elt.nil? || elt == '' }
+          next
+        else
+          @fieldset = Tb::FieldSet.new(*ary)
+          return @fieldset.header
+        end
+      end
+      @fieldset = Tb::FieldSet.new
+    end
+    return @fieldset.header
+  end
+
+  def index_from_field(f)
+    self.header
+    if @opt_n
+      raise "numeric field start from 1: #{f.inspect}" if /\A0+\z/ =~ f
+      raise "numeric field name expected: #{f.inspect}" if /\A(\d+)\z/ !~ f
+      $1.to_i - 1
+    else
+      @fieldset.index_from_field(f)
+    end
+  end
+
+  def field_from_index_ex(i)
+    raise ArgumentError, "negative index: #{i}" if i < 0
+    self.header
+    if @opt_n
+      if @fieldset.length <= i
+        @fieldset.add_fields(*(@fieldset.header.length..i).to_a.map {|j| "#{j+1}" })
+      end
+    end
+    @fieldset.field_from_index_ex(i)
+  end
+
+  def field_from_index(i)
+    raise ArgumentError, "negative index: #{i}" if i < 0
+    self.header
+    if @opt_n
+      return "#{i+1}"
+    end
+    @fieldset.field_from_index(i)
+  end
+
+  def shift
+    header
+    ary = @reader.shift
+    field_from_index_ex(ary.length-1) if ary && !ary.empty?
+    ary
+  end
+
+  def each
+    while ary = self.shift
+      yield ary
+    end
+    nil
+  end
+
+  def close
+    @reader.close
+  end
+
+  def fix_header(header)
+    h = {}
+    header.map {|s|
+      s ||= ''
+      if h[s]
+        s += "(2)" if /\(\d+\)\z/ !~ s
+        while h[s]
+          s = s.sub(/\((\d+)\)\z/) { n = $1.to_i; "(#{n+1})" }
+        end
+        s
+      end
+      h[s] = true
+      s
+    }
+  end
+end

data/lib/tb/record.rb

@@ -0,0 +1,129 @@
+# lib/tb/record.rb - record class for table library
+#
+# Copyright (C) 2010-2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+class Tb::Record
+  include Enumerable
+
+  def initialize(table, recordid)
+    @table = table
+    @recordid = recordid
+  end
+  attr_reader :table
+
+  def record_id
+    @recordid
+  end
+
+  def pretty_print(q) # :nodoc:
+    q.object_group(self) {
+      fs = @table.list_fields.reject {|f| self[f].nil? }
+      unless fs.empty?
+        q.text ':'
+        q.breakable
+      end
+      q.seplist(fs, nil, :each) {|f|
+        v = self[f]
+        q.group {
+          q.pp f
+          q.text '=>'
+          q.group(1) {
+            q.breakable ''
+            q.pp v
+          }
+        }
+      }
+    }
+  end
+  alias inspect pretty_print_inspect # :nodoc:
+
+  def has_field?(field)
+    @table.has_field?(field)
+  end
+
+  def [](field)
+    @table.get_cell(@recordid, field)
+  end
+
+  def []=(field, value)
+    @table.set_cell(@recordid, field, value)
+  end
+
+  def to_h
+    h = {}
+    @table.each_field {|f|
+      v = @table.get_cell(@recordid, f)
+      h[f] = v if !v.nil?
+    }
+    h
+  end
+
+  def to_h_with_reserved
+    h = {}
+    @table.each_field_with_reserved {|f|
+      v = @table.get_cell(@recordid, f)
+      h[f] = v if !v.nil?
+    }
+    h
+  end
+
+  def to_a
+    a = {}
+    @table.each_field {|f|
+      v = @table.get_cell(@recordid, f)
+      a << [f, v] if !v.nil?
+    }
+    a
+  end
+
+  def to_a_with_reserved
+    a = {}
+    @table.each_field_with_reserved {|f|
+      v = @table.get_cell(@recordid, f)
+      a << [f, v] if !v.nil?
+    }
+    a
+  end
+
+  def each
+    @table.each_field {|f|
+      v = @table.get_cell(@recordid, f)
+      yield [f, v] if !v.nil?
+    }
+    nil
+  end
+
+  def each_with_reserved
+    @table.each_field_reserved {|f|
+      v = @table.get_cell(@recordid, f)
+      yield [f, v] if !v.nil?
+    }
+    nil
+  end
+
+  def values_at(*fields)
+    fields.map {|f| self[f] }
+  end
+end

data/lib/tb/tsv.rb

@@ -0,0 +1,93 @@
+# lib/tb/tsv.rb - TSV related fetures for table library
+#
+# Copyright (C) 2010-2011 Tanaka Akira  <akr@fsij.org>
+# 
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+#  1. Redistributions of source code must retain the above copyright notice, this
+#     list of conditions and the following disclaimer.
+#  2. Redistributions in binary form must reproduce the above copyright notice,
+#     this list of conditions and the following disclaimer in the documentation
+#     and/or other materials provided with the distribution.
+#  3. The name of the author may not be used to endorse or promote products
+#     derived from this software without specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+# OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+# OF SUCH DAMAGE.
+
+require 'stringio'
+
+class Tb
+  def Tb.tsv_stream_input(tsv)
+    tsvreader = TSVReader.new(tsv)
+    while ary = tsvreader.shift
+      yield ary
+    end
+    nil
+  end
+
+  class TSVReader
+    def initialize(input)
+      if input.respond_to? :to_str
+        @input = StringIO.new(input)
+      else
+        @input = input
+      end
+    end
+
+    def shift
+      line = @input.gets
+      return nil if !line
+      line = line.chomp("\n")
+      line = line.chomp("\r")
+      line.split(/\t/, -1)
+    end
+
+    def close
+      @input.close
+    end
+  end
+
+  def Tb.tsv_stream_output(out)
+    gen = Object.new
+    gen.instance_variable_set(:@out, out)
+    def gen.<<(ary)
+      @out << Tb.tsv_fields_join(ary) << "\n"
+    end
+    yield gen
+  end
+
+  # :call-seq:
+  #   generate_tsv(out='', fields=nil) {|recordids| modified_recordids }
+  #   generate_tsv(out='', fields=nil)
+  #
+  def generate_tsv(out='', fields=nil, &block)
+    if fields.nil?
+      fields = list_fields
+    end
+    recordids = list_recordids
+    if block_given?
+      recordids = yield(recordids)
+    end
+    Tb.tsv_stream_output(out) {|gen|
+      gen << fields
+      recordids.each {|recordid|
+        gen << get_values(recordid, *fields)
+      }
+    }
+    out
+  end
+
+  def Tb.tsv_fields_join(values)
+    values.map {|v| v.to_s.gsub(/[\t\r\n]/, ' ') }.join("\t")
+  end
+end