extract 0.1.1 → 0.1.3

data/lib/extract/excel_formulas.rb

@@ -14,7 +14,7 @@ module Extract
     def max(*args)
       args.flatten.select { |x| x }.sort.reverse.first
     end
-    def vlookup(lookup_val,range,col_num)
+    def vlookup(lookup_val,range,col_num,*junk)
       range.each do |row|
         if row[0] == lookup_val
           return row[col_num-1]

data/lib/extract/export/ddl.rb

@@ -0,0 +1,8 @@
+module Extract
+  module Export
+    class Ddl
+      include FromHash
+      attr_accessor :tables
+    end
+  end
+end

data/lib/extract/export/table.rb

@@ -0,0 +1,48 @@
+module Extract
+  module Export
+    class Table
+      include FromHash
+      attr_accessor :name
+      fattr(:rows) { [] }
+
+      def cols
+        rows.map { |x| x.keys }.flatten.uniq
+      end
+
+      def quoted_col(col)
+        return col
+        if col.to_s[0..0] =~ /\d/
+          "\"#{col}\""
+        else
+          col
+        end
+      end
+
+      def create_table_sql
+        col_str = cols.map { |x| "  #{quoted_col(x)} varchar(255)" }.join(",\n")
+        "CREATE TABLE #{name} (
+#{col_str}
+);"
+      end
+
+      def inserts
+        res = []
+        rows.each do |row|
+          col_str = row.keys.map { |x| quoted_col(x) }.join(",")
+          val_str = row.values.map { |x| "'#{x}'" }.join(',')
+          str = "INSERT INTO #{name} (#{col_str}) VALUES (#{val_str});"
+          res << str
+        end
+        res
+      end
+
+      def sql_statements
+        [create_table_sql,inserts].flatten
+      end
+
+      def sql
+        sql_statements.join("\n")
+      end
+    end
+  end
+end

data/lib/extract/formula.treetop

@@ -57,6 +57,9 @@ grammar Formula
       def excel_value
         (text_value == "TRUE") ? true : false
       end
+      def deps
+        []
+      end
     }
   end
 

data/lib/extract/inline_def.rb

@@ -0,0 +1,38 @@
+module Extract
+  class InlineDef
+    include FromHash
+    attr_accessor :raw
+    def parse_raw_cell(cell)
+      if cell == '_'
+        nil
+      elsif cell =~ /^=/
+        cell
+      elsif cell =~ /[a-z]/i
+        cell
+      elsif cell =~ /[0-9]/i
+        cell.to_f
+      else
+        raise "dunno"
+      end
+    end
+    fattr(:rows) do
+      raw.strip.split("\n").map do |str|
+        str.strip.split(" ").map do |cell|
+          parse_raw_cell(cell)
+        end
+      end
+    end
+
+    fattr(:sheet) do
+      res = Sheet.new
+      letters = ("A".."Z").to_a
+      rows.each_with_index do |row,row_i|
+        row.each_with_index do |val,col_i|
+          cell = letters[col_i] + (row_i+1).to_s
+          res[cell] = val if val
+        end
+      end
+      res
+    end
+  end
+end

data/lib/extract/sheet.rb

@@ -17,6 +17,9 @@ module Extract
         res
       end
     end
+    def raw_value(c)
+      cells[c]
+    end
 
     def clear_cache!
       self.cache = {}
@@ -58,18 +61,19 @@ module Extract
 
 
     class << self
-      def load(file)
+      def load(file,sheet_name=nil)
         w = Roo::Excelx.new(file)
-        w.default_sheet = w.sheets.first
+        w.default_sheet = sheet_name || w.sheets.first
 
         sheet = Extract::Sheet.new
 
         ("A".."Z").each do |col|
           (1..100).each do |row|
-            val = if w.formula?(row,col)
+            cell_text = w.cell(row,col)
+            val = if cell_text.present? && w.formula?(row,col)
               "=" + w.formula(row,col).gsub(" ","")
             else
-              w.cell(row,col)
+              cell_text
             end
             loaded = w.cell(row,col)
             sheet["#{col}#{row}"] = val if val.present?
@@ -79,6 +83,10 @@ module Extract
 
         sheet
       end
+
+      def inline(str)
+        InlineDef.new(:raw => str).sheet
+      end
     end
 
   end

data/lib/extract/sheet_definition.rb

@@ -3,6 +3,8 @@ module Extract
     include FromHash
     attr_accessor :sheet
 
+    fattr(:tables) { Tables.new(:sheet_def => self) }
+
     def prev_letter(letter)
       r = ("A".."Z").to_a
       raise "bad letter #{letter}" unless r.index(letter)
@@ -19,19 +21,19 @@ module Extract
       res = {}
       (input_cells + output_cells).each do |c|
         n = left(c)
-        res[c] = sheet[n]
+        res[c] = sheet[n] if !sheet[n].kind_of?(Numeric) && sheet[n].to_s.strip != '' && sheet[n].to_s.strip != 'N/A'
+      end
+
+      each_other_basic do |c|
+        n = left(c)
+        res[c] = sheet[n] if !sheet[n].kind_of?(Numeric) && sheet[n].to_s.strip != '' && sheet[n].to_s.strip != 'N/A'
       end
+
       res
     end
     fattr(:output_cells) { [] }
     def output_cells=(arr)
-      @output_cells = arr.map do |c|
-        if c =~ /:/
-          Extract::Tree::Range.cells_in_range(c)
-        else
-          c
-        end
-      end.flatten
+      @output_cells = Extract.expand_cells(arr).uniq
     end
 
     fattr(:dep_map) do
@@ -48,6 +50,20 @@ module Extract
       res
     end
 
+    def deps(cell,ops={})
+      raw = sheet.deps(cell)
+      if ops[:table]
+        res = []
+        raw.each do |c|
+          t = tables.for_cell(c)
+          res << (t || c)
+        end
+        res.uniq.sort
+      else
+        raw
+      end
+    end
+
     fattr(:input_cells) do
       output_cells.map do |c|
         a = dep_map[c] || []
@@ -86,6 +102,9 @@ module Extract
     def [](c)
       sheet[c]
     end
+    def raw_value(c)
+      sheet.cells[c]
+    end
 
     def each_input
       input_cells.each do |cell|
@@ -99,7 +118,7 @@ module Extract
       end
     end
 
-    def each_other
+    def each_other_basic
       res = []
       bad = input_cells + output_cells
       sheet.cells.each do |k,v|
@@ -110,15 +129,21 @@ module Extract
 
       res.each do |c|
         d = sheet.deps(c)
-        yield c,sheet.cells[c],d if sheet.cells[c].present? && d.size > 0
+        yield c if sheet.cells[c].present? && d.size > 0
       end
+    end
 
+    def each_other
+      each_other_basic do |c|
+        d = sheet.deps(c)
+        yield c,cell_names[c],sheet[c],sheet.cells[c],d
+      end
     end
 
     class << self
-      def load(file,output)
+      def load(file,output,sheet_name=nil)
         res = new
-        res.sheet = Sheet.load(file)
+        res.sheet = Sheet.load(file,sheet_name)
         res.output_cells = output
         res
       end

data/lib/extract/table.rb

@@ -0,0 +1,67 @@
+module Extract
+  class Row
+    include FromHash
+    include Enumerable
+    attr_accessor :table, :cells
+
+    fattr(:value_hash) do
+      res = {}
+      raise "bad match" unless table.field_names.size == cells.size
+      table.field_names.each_with_index do |name,i|
+        cell = cells[i]
+        res[name] = cell.value
+      end
+      res
+    end
+
+    def [](k)
+      value_hash[k]
+    end
+
+    def each(&b)
+      value_hash.each(&b)
+    end
+
+    def present?
+      value_hash.values.any? { |x| x.present? }
+    end
+  end
+  class Table
+    include FromHash
+    attr_accessor :cell_range, :name, :sheet_def
+
+    def cells
+      Extract.expand_cells(cell_range)
+    end
+
+    def cell_objs
+      cells.map { |c| Cell.new(:sheet_def => sheet_def, :cell => c) }
+    end
+    def cell_row_hash
+      res = Hash.new { |h,k| h[k] = [] }
+      cell_objs.each do |c|
+        res[c.row] << c
+      end
+      res
+    end
+
+    def field_names
+      k = cell_row_hash.keys.min
+      cell_row_hash[k].map { |x| x.value }
+    end
+
+    def rows
+      cell_row_hash.values[1..-1].map { |a| Row.new(:table => self, :cells => a) }.select { |x| x.present? }
+    end
+
+    def sql_statements
+      res = Extract::Export::Table.new(:name => name)
+      rows.each do |row|
+        res.rows << row.value_hash
+      end
+      res.sql_statements
+    end
+
+    
+  end
+end

data/lib/extract/tables.rb

@@ -0,0 +1,39 @@
+module Extract
+  class Tables
+    include FromHash
+    attr_accessor :sheet_def
+    fattr(:tables) { {} }
+
+    def add(name,range)
+      self.tables[name] = Table.new(:cell_range => range, :name => name, :sheet_def => sheet_def)
+    end
+
+    fattr(:cell_map) do
+      res = {}
+      tables.each do |name,table|
+        table.cells.each do |c|
+          res[c] = name
+        end
+      end
+      res
+    end
+
+    def for_cell(c)
+      cell_map[c]
+    end
+
+    def [](c)
+      if c.to_s == 'all'
+        Table.new(:cell_range => "A1:D100", :name => "all", :sheet_def => sheet_def)
+      else
+        tables[c]
+      end
+    end
+    def each(&b)
+      tables.each(&b)
+    end
+    def values
+      tables.values
+    end
+  end
+end

data/spec/cell_spec.rb

@@ -0,0 +1,31 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe 'Cell' do
+  let(:sheet) do
+    res = Extract::Sheet.new
+    {"A1" => 1, "A2" => 2, "A3" => 3, "B1" => 4, "B2" => 5, "B3" => 6}.each do |k,v|
+      res[k] = v
+    end
+    res["C1"] = "=A1"
+    res['D1'] = "=C1"
+    res['E1'] = "=D1"
+
+    res['F1'] = "=A1+A2"
+    res
+  end
+
+  let(:cell) do
+    Extract::Cell.new(:sheet_def => sheet, :cell => "C1")
+  end
+
+  it 'value' do
+    cell.value.should == 1
+  end
+
+  it 'row' do
+    cell.row.should == 1
+  end
+  it 'col' do
+    cell.col.should == "C"
+  end
+end

data/spec/config/mongoid.yml

@@ -3,4 +3,11 @@ development:
     default:
       database: mongoid_dev
       hosts:
-        - localhost:27017
+        - localhost:27017
+test:
+  sessions:
+    default:
+      uri: mongodb://extract_test:extract_test@linus.mongohq.com:10038/extract_test
+      options:
+        skip_version_check: true
+        safe: true

data/spec/export/table_spec.rb

@@ -0,0 +1,45 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+class String
+  def ssf
+    gsub("\n"," ").gsub(/[ ]{2,99}/," ")
+  end
+end
+
+describe 'Export Table' do
+  describe "basic" do
+    let(:table) do
+      res = Extract::Export::Table.new(:name => "widgets")
+      res.rows << {:color => "Green", :price => 20}
+      res
+    end
+
+    it 'create table sql' do
+      exp = "CREATE TABLE widgets (
+        color varchar(255),
+        price varchar(255)
+      );"
+      table.create_table_sql.ssf.should == exp.ssf
+    end
+
+    it 'inserts' do
+      table.inserts.size.should == 1
+      table.inserts[0].ssf.should == "INSERT INTO widgets (color,price) VALUES ('Green','20');"
+    end
+  end
+
+  describe "needs quotes" do
+    let(:table) do
+      res = Extract::Export::Table.new(:name => "widgets")
+      res.rows << {"2B" => 14}
+      res
+    end
+
+    it 'create table' do
+      exp = 'CREATE TABLE widgets (
+        "2B" varchar(255)
+      );'
+      table.create_table_sql.ssf.should == exp.ssf
+    end
+  end
+end