data_list_converter 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f397cef319fe34bff9143714c69003b6e9f2cb2f
+  data.tar.gz: 96708c56c0bfd2a4e48ebff454ec004c3becda34
+SHA512:
+  metadata.gz: 20f9d84f520b6039d6c4a2b7bb1c7e9367a16a18b30818170a98637742bccf9524d5a2fefaa919730eb4b736e66571f13ab9acea08ade8c29200d996b9a7f80a
+  data.tar.gz: 957d4cc8c98c9dc7cad99814ee4acaaa18cf222481c043926649b86241d4e5deb4db74d31e9631f97a428e10952a46ac152759a9ee24acaeda54a774cac11c62

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+*.gem

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2015 linjunhalida
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+# Data List Converter
+
+Data List Converter is a tool to convert data between different formats.
+
+Example:
+
+```ruby
+data = [{name: 'James', age: '22'}, {name: 'Bob', age: '33'}]
+DataListConverter.convert(:item_data, :table_data, data)
+# => [["name", "age"], ["James", "22"], ["Bob", "33"]] 
+DataListConverter.convert(:item_data, :csv_file, data, csv_file: {filename: 'result.csv'})
+DataListConverter.convert(:item_data, :xls_file, data, csv_file: {filename: 'result.csv'})
+
+DataListConverter.convert(:csv_file, :item_data, {filename: 'result.csv'}) == data
+```
+
+You can also add filter to this process:
+
+```ruby
+filter = :limit
+filter = {limit: {size: 2}}
+filter = [{limit: {size: 12}}, {count: {size: 4}}]
+convert(:item_iterator, :table_data, iter, table_iterator: {filter: filter})
+```
+
+Please read [the source code](https://github.com/halida/data_list_converter/blob/master/lib/data_list_converter.rb) for more information,
+also you can check [test examples](https://github.com/halida/data_list_converter/blob/master/test/data_list_converter_test.rb).
+
+## Data Types
+
+- **item_data** like: `[{name: 'James', age: '22'}, ...]`
+- **item_iterator** iterator for item_data, used like: iter.call{|item| out << item}
+- **table_data** like: `[["name", "age"], ["James", "22"], ["Bob", "33"], ...]`
+- **table_iterator** iterator for table_data
+- **csv_file** file in csv format
+- **xls_file** file in excel format
+- **multi_sheet_data** like: `{'sheet1' => [columns, row, row, ...], 'sheet2' => [columns, row, row, ...]}`
+- **multi_sheet_iterator** like: `{'sheet1' => table_iterator1, 'sheet2' => table_iterator2}`
+- **records** ActiveRecord records, usage: `DataListConverter.convert(:records, :item_data, query, item_iterator: {columns: [:name, :age]})`
+
+
+## Filters
+
+**item_iterator/table_iterator limit**: limit item_iterator result counts, usage: `DataListConverter.convert(:item_data, :table_data, item_data, item_iterator: {filter: {limit: {size: 2}}})`
+
+**item_iterator count**: count item_iterator items, usage: `DataListConverter.convert(:xls_file, :item_data, {filename: 'result.xls'}, item_iterator: {filter: {count: {size: 10}}})`, it will print current item counts every `size`.
+
+## Extend
+
+You can add your data types and filters, example:
+
+```ruby
+DataListConverter.register_converter(
+  :records, :item_iterator, lambda { |records, options|
+    columns = options[:columns]
+    lambda { |&block|
+      records.find_each do |record|
+        item = columns.map do |column|
+          [column.first.to_sym, record.send(column[1])]
+        end.to_h
+        block.call(item)
+      end
+    }
+  })
+
+DataListConverter.register_filter(
+  :item_iterator, :limit, lambda { |proc, options|
+    limit_size = options[:size] || 10
+    lambda { |&block|
+      limit = 0
+      proc.call do |item|
+        block.call(item)
+        limit += 1
+        break if limit >= limit_size
+      end
+    }
+  })
+```

data/Rakefile

@@ -0,0 +1,18 @@
+$:.push File.expand_path("lib", __FILE__)
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+task :default => :test
+
+task :test do
+  load 'test/data_list_converter_test.rb'
+end
+
+task :build do
+  sh 'gem build data_list_converter.gemspec'
+end
+
+task :upload do
+  sh 'gem push *.gem'
+end

data/data_list_converter.gemspec

@@ -0,0 +1,22 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "data_list_converter"
+  s.version     = "0.1"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["linjunhalida"]
+  s.email       = ["linjunhalida@gmail.com"]
+  s.homepage    = "http://github.com/halida/data_list_converter"
+  s.summary     = "convert data between different formats"
+  s.description = "Data List Converter is a tool to convert data between different formats."
+  s.licenses    = ["MIT"]
+
+  s.add_development_dependency 'rake', '~> 10.4'
+  s.add_development_dependency 'minitest', '~> 5.7'
+  s.add_development_dependency 'spreadsheet', '~> 1.0'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/data_list_converter.rb

@@ -0,0 +1,353 @@
+require 'csv'
+
+# this class is used for convert data between types,
+# so we can use it to easily export and import data.
+# 
+class DataListConverter
+  CONVERTERS = {}
+  FILTERS = {}
+
+  class << self
+
+    def types
+      CONVERTERS.keys.flatten.uniq.sort
+    end
+
+    def register_converter(from_type, to_type, method)
+      @route_map = nil # clear cache
+      CONVERTERS[[from_type, to_type]] = method
+    end
+
+    def register_filter(type, name, method)
+      FILTERS[type] ||= {}
+      FILTERS[type][name] = method
+    end
+
+    # Example:
+    # convert(:item_iterator, :item_data, iter)
+    # convert(:item_iterator, :csv_file, iter, csv_file: {filename: 'result.csv'})
+    # convert(:csv_file, :item_data, {filename: 'result.csv'})
+    #
+    # can add filter:
+    # filter = :limit
+    # filter = {limit: {size: 2}}
+    # filter = [{limit: {size: 12}}, {count: {size: 4}}]
+    # convert(:item_iterator, :table_data, iter, table_iterator: {filter: filter})
+    def convert(from_type, to_type, from_value, options={})
+      methods = []
+      add_filter = lambda { |type|
+        filters = (options[type] || {}).delete(:filter)
+        return unless filters
+        methods += normalize_filters(type, filters)
+      }
+
+      route = find_route(from_type, to_type)
+      add_filter.call(route[0])
+
+      (0..(route.length-2)).map do |i|
+        from_type, to_type = route[i], route[i+1]
+        method = CONVERTERS[[from_type, to_type]]
+        raise "cannot find converter #{from_type} -> #{to_type}" unless method
+        methods.push([method, options[to_type] || {}])
+        add_filter.call(to_type)
+      end
+
+      methods.inject(from_value) do |v, method|
+        method, args = method
+        method.call(v, args)
+      end
+    end
+
+    def normalize_filters(type, filters)
+      # filter list as array
+      filters = [filters] unless filters.kind_of?(Array)
+      filters.map do |v|
+        # fix filter arguments
+        case v
+        # {:limit, {count: 12}} => [:limit, {count: 12}]
+        when Hash; v.first
+        # :debug => [:debug, {}]
+        when Symbol, String; [v, {}]
+        else; v
+        end
+      end.map do |name, args|
+        method = FILTERS[type][name] rescue raise("cannot find method for type #{type} filter #{name}")
+        [method, args]
+      end
+    end
+
+    # One type of data can be converted into any other types,
+    # we have a list of convert methods: CONVERTERS
+    # If we want to convert between types, like: convert item_data into csv_file,
+    # we need find all the intermidate data type, like: [:item_data, :item_iterator, :table_iterator, :csv_file]
+    def find_route(from_type, to_type)
+      raise Exception, "from_type should not equal to to_type: #{from_type}" if from_type == to_type
+      out = find_next_node([], [from_type], route_map, to_type)
+      raise Exception, "Route not found: #{from_type} -> #{to_type}" unless out
+      out
+    end
+
+    # iterate through the type convert graph, and find the route
+    def find_next_node (out, nodes, map, end_node)
+      nodes.each do |node|
+        return out + [node] if node == end_node
+        next unless next_nodes = map[node]
+
+        new_map = map.dup
+        new_map.delete(node)
+        result = find_next_node(out + [node], next_nodes, new_map, end_node)
+        return result if result
+      end
+      nil
+    end
+    private :find_next_node
+
+    # convert adjacency list into quick lookup hash
+    def route_map
+      @route_map ||= \
+      begin
+        CONVERTERS.keys.
+          inject({}) do |map, item|
+          map[item.first] ||= []
+          map[item.first] += [item[1]]
+          map
+        end
+      end
+    end
+
+  end
+end
+
+# register types
+
+# item_data: [{name: 'xx', value: 12}, ...]
+# table_data: [['name', 'value'], ['xx', '12'], ..]
+# item_iterator and table_iterator are iterators which yield each row
+class DataListConverter
+  self.register_converter(
+    :item_iterator, :table_iterator, lambda{ |proc, options|
+      lambda { |&block|
+        columns = nil
+        proc.call do |item|
+          unless columns
+            columns = item.keys.map(&:to_sym)
+            block.call(columns.map(&:to_s))
+          end
+          # item_iterator key can be symbol or string
+          block.call(columns.map{ |c| item[c] || item[c.to_s] })
+        end
+      }
+    })
+  self.register_converter(
+    :table_iterator, :item_iterator, lambda{ |proc, options|
+      lambda {|&block|
+        columns = nil
+        proc.call do |row|
+          unless columns
+            columns = row.map(&:to_sym)
+          else
+            block.call(columns.zip(row).to_h)
+          end
+        end
+      }
+    })
+
+  def self.iterator_to_data(proc, options={})
+    out = []
+    proc.call { |d| out << d }
+    out
+  end
+  self.register_converter(
+    :item_iterator, :item_data, self.method(:iterator_to_data))
+  self.register_converter(
+    :table_iterator, :table_data, self.method(:iterator_to_data))
+
+  def self.data_to_iterator(data, options={})
+    lambda { |&block|
+      data.each do |d|
+        block.call(d)
+      end
+    }
+  end
+  self.register_converter(
+    :item_data, :item_iterator, self.method(:data_to_iterator))
+  self.register_converter(
+    :table_data, :table_iterator, self.method(:data_to_iterator))
+end
+
+# records
+class DataListConverter
+  self.register_converter(
+    :records, :item_iterator, lambda { |records, options|
+      columns = options[:columns]
+      lambda { |&block|
+        records.find_each do |record|
+          item = columns.map do |column|
+            [column.first.to_sym, record.send(column[1])]
+          end.to_h
+          block.call(item)
+        end
+      }
+    })
+end
+
+# csv_file
+class DataListConverter
+  self.register_converter(
+    :csv_file, :table_iterator, lambda { |input, options|
+      lambda { |&block|
+        CSV.open(input[:filename]) do |csv|
+          csv.each do |row|
+            block.call(row)
+          end
+        end
+      }
+    })
+  self.register_converter(
+    :table_iterator, :csv_file, lambda { |proc, options|
+      CSV.open(options[:filename], 'wb', force_quotes: true) do |csv|
+        proc.call do |row|
+          csv << row
+        end
+      end
+    })
+end
+
+# xls_file only works when installed spreadsheet gem
+# multi_sheet_data = {
+#   'sheet1' => [columns, row, row, ...],
+#   'sheet2' => [columns, row, row, ...],
+# }
+begin
+  require 'spreadsheet'
+
+  class DataListConverter
+    self.register_converter(
+      :xls_file, :table_iterator, lambda { |input, options|
+        lambda { |&block|
+          book = Spreadsheet.open(input[:filename])
+          sheet = book.worksheet input[:sheet] || 0
+          sheet.each do |row|
+            block.call(row.to_a)
+          end
+        }
+      })
+    self.register_converter(
+      :table_iterator, :xls_file, lambda { |proc, options|
+        book = Spreadsheet::Workbook.new
+        sheet = book.create_worksheet(name: (options[:sheet] || "Sheet1"))
+        i = 0
+        proc.call do |row|
+          sheet.row(i).push *row
+          i += 1
+        end
+        book.write(options[:filename])
+        options[:filename]
+      })
+
+    self.register_converter(
+      :multi_sheet_iterator, :multi_sheet_data, lambda { |data, options|
+        data.map do |k, iter|
+          data = self.convert(:table_iterator, :table_data, iter)
+          [k, data]
+        end.to_h
+      })
+    self.register_converter(
+      :multi_sheet_data, :multi_sheet_iterator, lambda { |data, options|
+        data.map do |k, v|
+          iter = self.convert(:table_data, :table_iterator, v)
+          [k, iter]
+        end.to_h
+    })
+    self.register_converter(
+      :multi_sheet_iterator, :xls_file, lambda { |data, options|
+        book = Spreadsheet::Workbook.new
+        data.each do |name, table_iterator|
+          sheet = book.create_worksheet(name: name)
+          i = 0
+          table_iterator.call do |row|
+            sheet.row(i).concat(row)
+            i += 1
+          end
+        end
+        filename = options[:filename]
+        book.write(filename)
+        filename
+      })
+    self.register_converter(
+      :xls_file, :multi_sheet_iterator, lambda { |data, options|
+        book = Spreadsheet.open(data[:filename])
+        book.worksheets.map do |sheet|
+          iterator = lambda { |&block|
+            sheet.each do |row| 
+              block.call(row.to_a)
+            end
+          }
+          [sheet.name, iterator]
+        end.to_h
+      })
+  end
+rescue LoadError
+  nil
+end
+
+
+# filters
+class DataListConverter
+  self.register_filter(
+    :table_iterator, :remove_debug, lambda{ |proc, options|
+      lambda { |&block|
+        columns = nil
+        debug_index = nil
+        proc.call do |row|
+          unless columns
+            columns = row
+            debug_index = columns.index('debug')
+          end
+          block.call(row[0..(debug_index-1)])
+        end
+      }
+    })
+
+  def self.iterator_limit(proc, options)
+    limit_size = options[:size] || 10
+    lambda { |&block|
+      limit = 0
+      proc.call do |item|
+        block.call(item)
+        limit += 1
+        break if limit >= limit_size
+      end
+    }
+  end
+  self.register_filter(
+    :item_iterator, :limit, self.method(:iterator_limit))
+  self.register_filter(
+    :table_iterator, :limit, self.method(:iterator_limit))
+
+  self.register_filter(
+    :item_iterator, :count, lambda{ |proc, options|
+      count = 0
+      size = options[:size] || 100
+      msg_format = options[:msg] || "on %{count}"
+      total_format = options[:total] || "total: %{total}"
+      out = options[:out] || STDOUT
+      total = 1
+      lambda { |&block|
+        proc.call do |item|
+          if item.keys == [:total]
+            total = item[:total]
+            out.write(total_format % {total: total})
+            out.write("\n")
+          else
+            block.call(item)
+            count += 1
+            msg = msg_format % {count: count, percent: (count / total.to_f * 100).round(2)}
+            out.write(msg + "\n") if count % size == 0
+          end
+        end
+      }
+    })
+end
+
+

data/test/data_list_converter_test.rb

@@ -0,0 +1,118 @@
+require 'minitest/pride'
+require 'minitest/autorun'
+
+require 'data_list_converter'
+
+ITEM_DATA = [
+  {name: "James", score: '12'},
+  {name: "Bob", score: '33'},
+]
+
+TABLE_DATA = [
+  ['name', 'score'],
+  ['James', '12'],
+  ['Bob', '33']
+]
+
+CSV_DATA = %{
+"name","score"
+"James","12"
+"Bob","33"
+}.strip
+
+describe DataListConverter do
+  describe :type do
+    it 'works' do
+      DataListConverter.types.must_equal(
+        [:csv_file, :item_data, :item_iterator, :multi_sheet_data, :multi_sheet_iterator,
+         :records, :table_data, :table_iterator, :xls_file])
+    end
+  end
+
+  describe :convert do
+    before :all do
+      @c = DataListConverter
+    end
+    
+    it 'works' do
+      @c.convert(:item_data, :table_data, ITEM_DATA).must_equal TABLE_DATA
+    end
+
+    it 'has parameters' do
+      begin
+        @c.convert(:item_data, :csv_file, ITEM_DATA, csv_file: {filename: "test.csv"})
+        File.read("test.csv").strip.must_equal CSV_DATA
+        @c.convert(:csv_file, :item_data, {filename: "test.csv"}).must_equal(ITEM_DATA)
+      ensure
+        FileUtils.rm_f("test.csv")
+      end
+    end
+
+    it 'has filters' do
+      filter = :limit
+      item_data = [{name: "james"}] * 20
+      result = @c.convert(:item_data, :table_data, item_data,
+                          item_iterator: {filter: filter})
+      result.must_equal([["name"]] + [["james"]]*10)
+      
+      filter = {limit: {size: 2}}
+      item_data = [{name: "james"}] * 10
+      result = @c.convert(:item_data, :table_data, item_data,
+                          item_iterator: {filter: filter})
+      result.must_equal [["name"], ["james"], ["james"]]
+
+      string = StringIO.new
+      filter = [{limit: {size: 12}}, {count: {size: 4, out: string, msg: ".%{count}"}}]
+      item_data = [{name: "james"}] * 20
+      result = @c.convert(:item_data, :table_data, item_data,
+                          item_iterator: {filter: filter})
+      result.must_equal([["name"]] + [["james"]]*12)
+      string.string.must_equal ".4\n.8\n.12\n"
+    end
+
+    it 'has type xls_file' do
+      begin
+        @c.convert(:item_data, :xls_file, ITEM_DATA, xls_file: {filename: "test.xls"})
+        @c.convert(:xls_file, :item_data, {filename: "test.xls"}).must_equal ITEM_DATA
+
+        multi_sheet = {
+          "sheet1" => [['name'], ['james'], ['bob']],
+          "sheet2" => [['value'], ['21'], ['12']],
+        }
+        @c.convert(:multi_sheet_data, :xls_file, multi_sheet, xls_file: {filename: 'test.xls'})
+        @c.convert(:xls_file, :multi_sheet_data, {filename: 'test.xls'}).must_equal multi_sheet
+      ensure
+        FileUtils.rm_f("test.xls")
+      end
+    end
+  end
+
+  describe :filters do
+    it 'limit iterator' do
+      filter = {limit: {size: 2}}
+      item_data = [{name: "james"}] * 10
+      result = DataListConverter.convert(
+        :item_data, :table_data, item_data,
+        item_iterator: {filter: filter})
+      result.must_equal [["name"], ["james"], ["james"]]
+    end
+
+    it 'count item_iterator' do
+      iter = lambda { |&block|
+        total = 10000
+        block.call(total: total)
+        (1..total-1).each do |i|
+          block.call(id: i, value: i*i)
+        end
+      }
+      string = StringIO.new
+      filter = {count: {size: 4000,
+                        out: string,
+                        msg: "%{percent}%"}}
+      result = DataListConverter.convert(
+        :item_iterator, :table_data, iter,
+        item_iterator: {filter: filter})
+      string.string.split("\n").must_equal ['total: 10000', '40.0%', '80.0%']
+    end
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: data_list_converter
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- linjunhalida
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.7'
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Data List Converter is a tool to convert data between different formats.
+email:
+- linjunhalida@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- MIT-LICENSE
+- README.md
+- Rakefile
+- data_list_converter.gemspec
+- lib/data_list_converter.rb
+- test/data_list_converter_test.rb
+homepage: http://github.com/halida/data_list_converter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: convert data between different formats
+test_files:
+- test/data_list_converter_test.rb