hakoy 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f85a1137bd3b13e70498749a0081aae6a5dd5afc
-  data.tar.gz: 82525465fd4bf796153c99e9efa47bd197449062
+  metadata.gz: edfb149296a98760f3892625b06e5978897af7d7
+  data.tar.gz: 2d3a371ab815a7b138a246359b6ae14a0f58a450
 SHA512:
-  metadata.gz: a81c90bbfe9b5b344d4ed7dcde2e8a646c9e978cc8578048ade18f44f5b8013adee79308b922164a0d68df9564fb301e434fe85d92b480be9ff10dd41d1375d2
-  data.tar.gz: 28d15174940c4a86cf3f4bde518dd490fe482d573e0826ec8693509eea79474d1922c10996e11d422aaac590a157c6913832303d9fc6e0370359bf5b8731944d
+  metadata.gz: 88fcaf569cff2a9a8f7719db72720636fe28fb1ec4bd6c9fa7c96d4b27a482087fbb38ed21b7b6c310759f1a864f64923b4736a4371d826196a5b6056d6356e1
+  data.tar.gz: 48f1c3c541c6ade97f18af3098ae461f687abca4e66166d48c7682fcf42caf2478656b470ce624e0881316d78a2c3924932730c018ba247bc310c082099d4577

data/README.md

@@ -1,5 +1,7 @@
 # Hakoy
 
+[![Code Climate](https://codeclimate.com/github/he9lin/hakoy.png)](https://codeclimate.com/github/he9lin/hakoy)
+
 Parse and organize CSV data into timestamp-sliced directories.
 
 

data/lib/hakoy/append_strategy.rb

@@ -0,0 +1,19 @@
+module Hakoy
+  class AppendStrategy
+    def append_row_to_file(file_path, row_hash)
+      memory[file_path] << row_hash
+    end
+
+    def finalize!(uid_key)
+      memory.each do |file_path, rows_hash|
+        FileAppender.(file_path, rows_hash, uid_key:  uid_key)
+      end
+    end
+
+    private
+
+    def memory
+      @_memory ||= Hash.new { |h, k| h[k] = [] }
+    end
+  end
+end

data/lib/hakoy/ext/array.rb

@@ -0,0 +1,11 @@
+class Array
+  def self.wrap(object)
+    if object.nil?
+      []
+    elsif object.respond_to?(:to_ary)
+      object.to_ary || [object]
+    else
+      [object]
+    end
+  end
+end

data/lib/hakoy/file_appender/csv.rb

@@ -3,35 +3,59 @@ module Hakoy
     module Csv
       extend self
 
-      def call(file_path, row_hash, opts={})
-        uid_key = opts.fetch(:uid_key) { 'id' }
+      module DuplicatesFilter
+        class << self
+          def call(file_path, rows_hash, uid_key)
+            results = []
 
-        file_exists = File.exists?(file_path)
+            check_duplidate = -> (row) do
+              rows_hash.each do |row_hash|
+                unless row[uid_key] == row_hash[uid_key]
+                  results << row_hash
+                end
+              end
+            end
 
-        if file_exists
-          when_not_a_duplicate(file_path, row_hash, uid_key) do
-            append_to_csv_file(file_path, row_hash.values)
+            CSV.foreach(file_path, headers: true, &check_duplidate)
+            results
           end
-        else
-          append_to_csv_file(file_path, row_hash.keys, row_hash.values)
         end
       end
 
-      private
+      def call(file_path, rows_hash, opts={})
+        uid_key     = opts.fetch(:uid_key) { 'id' }
+        file_exists = File.exists?(file_path)
+        rows_hash   = Array.wrap(rows_hash)
+
+        return if rows_hash.empty?
 
-      def append_to_csv_file(file_path, *rows)
         CSV.open(file_path, 'a') do |to_file|
-          rows.each {|r| to_file << r}
+          append_row_hash_values = -> (row_hash) do
+            append_to_csv_file(to_file, row_hash.values)
+          end
+
+          if file_exists
+            when_not_a_duplicate(file_path, rows_hash, uid_key, &append_row_hash_values)
+          else
+            # Add header for new file and no need to check duplicates
+            header_hash = rows_hash[0].keys
+            append_to_csv_file to_file, header_hash
+            rows_hash.each(&append_row_hash_values)
+          end
         end
       end
 
-      def when_not_a_duplicate(file_path, row_hash, uid_key, &block)
-        is_duplicate = false
-        check_duplidate = -> (row) {
-          is_duplicate = true if row[uid_key] == row_hash[uid_key]
-        }
-        CSV.foreach(file_path, headers: true, &check_duplidate)
-        block.call unless is_duplicate
+      private
+
+      def append_to_csv_file(to_file, *rows)
+        rows.each {|r| to_file << r}
+      end
+
+      def when_not_a_duplicate(file_path, rows_hash, uid_key, &block)
+        unique_rows_hash = DuplicatesFilter.(file_path, rows_hash, uid_key)
+        unique_rows_hash.each do |row_hash|
+          block.call(row_hash)
+        end
       end
     end
   end

data/lib/hakoy/file_appender.rb

@@ -2,14 +2,15 @@ module Hakoy
   module FileAppender
     extend self
 
-    def append(file_path, row_hash, opts={})
-      dir     = File.dirname(file_path)
-      extname = File.extname(file_path)
+    def append(file_path, rows_hash, opts={})
+      dir      = File.dirname(file_path)
+      extname  = File.extname(file_path)
+      strategy = opts.delete(:strategy)
 
       ensure_dir_exist(dir)
 
-      strategy = find_strategy(extname)
-      strategy.(file_path, row_hash, opts)
+      strategy ||= find_strategy(extname)
+      strategy.(file_path, rows_hash, opts)
     end
     alias :call :append
 

data/lib/hakoy/file_iterator.rb

@@ -2,9 +2,10 @@ module Hakoy
   module FileIterator
     extend self
 
-    def call(file, &block)
+    def call(file, opts={}, &block)
       extname = File.extname(file)
-      find_iterator(extname).(file, &block)
+      iterator = opts.fetch(:iterator) { find_iterator(extname) }
+      iterator.(file, &block)
     end
 
     private

data/lib/hakoy/version.rb

@@ -1,3 +1,3 @@
 module Hakoy
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/hakoy.rb

@@ -3,6 +3,8 @@ require 'csv'
 
 require_relative "hakoy/version"
 require_relative "hakoy/ext/hash"
+require_relative "hakoy/ext/array"
+require_relative "hakoy/append_strategy"
 require_relative "hakoy/file_iterator"
 require_relative "hakoy/timestamp_path"
 require_relative "hakoy/row_normalizer"
@@ -18,21 +20,23 @@ module Hakoy
     DEFAULT_UID_KEY       = 'id'
 
     def initialize(conf)
-      @timestamp_key  = conf.fetch(:timestamp_key)
-      @db_dir         = conf.fetch(:db_dir)
-      @output_format  = conf.fetch(:output_format) { DEFAULT_OUTPUT_FORMAT }
-      @uid_key        = conf.fetch(:uid_key) { DEFAULT_UID_KEY }
-      required_keys   = conf.fetch(:required_keys)
-
-      @timestamp_path = TimestampPath.new
-      @row_normalizer = RowNormalizer.new(
-        required_keys: required_keys, uid_key: @uid_key)
+      @timestamp_key   = conf.fetch(:timestamp_key)
+      @db_dir          = conf.fetch(:db_dir)
+      @output_format   = conf.fetch(:output_format)   { DEFAULT_OUTPUT_FORMAT }
+      @uid_key         = conf.fetch(:uid_key)         { DEFAULT_UID_KEY       }
+      @file_iterator   = conf.fetch(:file_iterator)   { FileIterator          }
+      @append_strategy = conf.fetch(:append_strategy) { AppendStrategy.new    }
+      required_keys    = conf.fetch(:required_keys)
+
+      @timestamp_path  = TimestampPath.new
+      @row_normalizer  = RowNormalizer.new(
+        required_keys: required_keys, uid_key: @uid_key
+      )
     end
 
     def store(file)
-      FileIterator.(file) do |row_hash|
-        store_row(row_hash)
-      end
+      @file_iterator.(file) { |row_hash| store_row(row_hash) }
+      finalize_store!
     end
 
     private
@@ -41,7 +45,7 @@ module Hakoy
       file_path            = build_file_path(row_hash)
       normalized_row_hash  = normalize_row_hash(row_hash)
 
-      append_file(file_path, normalized_row_hash)
+      append_row_to_file(file_path, normalized_row_hash)
     end
 
     def build_file_path(row_hash)
@@ -54,8 +58,12 @@ module Hakoy
       @row_normalizer.normalize(row_hash)
     end
 
-    def append_file(file_path, row_hash)
-      FileAppender.(file_path, row_hash, uid_key: @uid_key)
+    def append_row_to_file(file_path, row_hash)
+      @append_strategy.append_row_to_file(file_path, row_hash)
+    end
+
+    def finalize_store!
+      @append_strategy.finalize!(@uid_key)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hakoy
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Lin He
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-30 00:00:00.000000000 Z
+date: 2014-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -84,6 +84,8 @@ files:
 - bin/rspec
 - hakoy.gemspec
 - lib/hakoy.rb
+- lib/hakoy/append_strategy.rb
+- lib/hakoy/ext/array.rb
 - lib/hakoy/ext/hash.rb
 - lib/hakoy/file_appender.rb
 - lib/hakoy/file_appender/csv.rb