hakoy 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edfb149296a98760f3892625b06e5978897af7d7
-  data.tar.gz: 2d3a371ab815a7b138a246359b6ae14a0f58a450
+  metadata.gz: 78043bf9756159ef93266bb984468e291cbfc086
+  data.tar.gz: c4213e97fd70bc9c3e5bdf4c3bfd5bff79e77fc5
 SHA512:
-  metadata.gz: 88fcaf569cff2a9a8f7719db72720636fe28fb1ec4bd6c9fa7c96d4b27a482087fbb38ed21b7b6c310759f1a864f64923b4736a4371d826196a5b6056d6356e1
-  data.tar.gz: 48f1c3c541c6ade97f18af3098ae461f687abca4e66166d48c7682fcf42caf2478656b470ce624e0881316d78a2c3924932730c018ba247bc310c082099d4577
+  metadata.gz: 60e31277315cbcac5559c6e7c885621711ace2802b667601451fd871bda7752d972f9d270ba6bbf9d3bf5da6113ac57c3929cb4341050e5bc96be25ea9a24a98
+  data.tar.gz: 60c43d0529f95632b27e4a0ce4d5f747cf1fdbd75bf97813ac098bccf79babfb0c49e0dff34de72df3cd41bbfcef27fc61af4309de96b78c19d0a45e68d43d69

data/README.md

@@ -24,8 +24,10 @@ conf = {
 Hakoy.('data/order.csv', conf)
 ```
 
+It creates and organizes directories and files using timestamps. Below is a sample screenshot.
+
+![screen shot 2014-06-12 at 12 13 34 pm](https://cloud.githubusercontent.com/assets/79277/3262506/0e4dc94c-f266-11e3-8974-db35186cbebd.png)
+
 ## TODO
 
-* Queue up multiple rows to write to a file; currently it does a file
-  open/close for every row.
 * Better unique key generation algorithm. It is too primitive now.

data/lib/hakoy/append_strategy.rb

@@ -4,9 +4,12 @@ module Hakoy
       memory[file_path] << row_hash
     end
 
-    def finalize!(uid_key)
+    def finalize!(opts)
+      uid_key      = opts.fetch(:uid_key)
+      keys_mapping = opts.fetch(:keys_mapping)
+
       memory.each do |file_path, rows_hash|
-        FileAppender.(file_path, rows_hash, uid_key:  uid_key)
+        FileAppender.(file_path, rows_hash, uid_key: uid_key, keys_mapping: keys_mapping)
       end
     end
 

data/lib/hakoy/file_appender/csv.rb

@@ -23,23 +23,26 @@ module Hakoy
       end
 
       def call(file_path, rows_hash, opts={})
-        uid_key     = opts.fetch(:uid_key) { 'id' }
-        file_exists = File.exists?(file_path)
-        rows_hash   = Array.wrap(rows_hash)
+        uid_key      = opts.fetch(:uid_key) { 'uid' }
+        keys_mapping = opts.fetch(:keys_mapping) # An array
+        file_exists  = File.exists?(file_path)
+        rows_hash    = Array.wrap(rows_hash)
+        keys         = keys_mapping.keys
+        header_keys  = keys_mapping.values
 
         return if rows_hash.empty?
 
         CSV.open(file_path, 'a') do |to_file|
           append_row_hash_values = -> (row_hash) do
-            append_to_csv_file(to_file, row_hash.values)
+            append_to_csv_file(to_file, row_hash.values_at(*header_keys))
           end
 
           if file_exists
             when_not_a_duplicate(file_path, rows_hash, uid_key, &append_row_hash_values)
           else
             # Add header for new file and no need to check duplicates
-            header_hash = rows_hash[0].keys
-            append_to_csv_file to_file, header_hash
+            header_hash = rows_hash[0].keys.map {|key| keys_mapping.key(key) }
+            append_to_csv_file to_file, keys
             rows_hash.each(&append_row_hash_values)
           end
         end

data/lib/hakoy/row_normalizer.rb

@@ -2,6 +2,12 @@ module Hakoy
   class RowNormalizer
     MissingRequiredKeysError = Class.new(StandardError)
 
+    module GenerateUniqueId
+      def self.call(hash)
+        hash.values.map(&:to_s).join
+      end
+    end
+
     def initialize(opts)
       @uid_key = opts.fetch(:uid_key)
       @required_keys = opts.fetch(:required_keys).dup.freeze
@@ -22,7 +28,7 @@ module Hakoy
     end
 
     def generate_unique_id(hash)
-      hash.values.map(&:to_s).join
+      GenerateUniqueId.(hash)
     end
 
     def assert_has_required_keys!(hash, required_keys)

data/lib/hakoy/version.rb

@@ -1,3 +1,3 @@
 module Hakoy
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/hakoy.rb

@@ -17,20 +17,23 @@ module Hakoy
 
   class Proxy
     DEFAULT_OUTPUT_FORMAT = 'csv'
-    DEFAULT_UID_KEY       = 'id'
+    DEFAULT_UID_KEY       = 'uid'
+    REQUIED_KEYS          = %w(customer product timestamp price quantity order_id)
 
     def initialize(conf)
-      @timestamp_key   = conf.fetch(:timestamp_key)
-      @db_dir          = conf.fetch(:db_dir)
-      @output_format   = conf.fetch(:output_format)   { DEFAULT_OUTPUT_FORMAT }
-      @uid_key         = conf.fetch(:uid_key)         { DEFAULT_UID_KEY       }
-      @file_iterator   = conf.fetch(:file_iterator)   { FileIterator          }
-      @append_strategy = conf.fetch(:append_strategy) { AppendStrategy.new    }
-      required_keys    = conf.fetch(:required_keys)
+      @timestamp_key         = conf.fetch(:timestamp_key)
+      @db_dir                = conf.fetch(:db_dir)
+      @output_format         = conf.fetch(:output_format)   { DEFAULT_OUTPUT_FORMAT }
+      @uid_key               = conf.fetch(:uid_key)         { DEFAULT_UID_KEY       }
+      @file_iterator         = conf.fetch(:file_iterator)   { FileIterator          }
+      @append_strategy       = conf.fetch(:append_strategy) { AppendStrategy.new    }
+      @required_keys_mapping = conf.fetch(:required_keys_mapping)
+
+      assert_required_keys_present!(@required_keys_mapping)
 
       @timestamp_path  = TimestampPath.new
       @row_normalizer  = RowNormalizer.new(
-        required_keys: required_keys, uid_key: @uid_key
+        required_keys: @required_keys_mapping.values, uid_key: @uid_key
       )
     end
 
@@ -41,6 +44,13 @@ module Hakoy
 
     private
 
+    def assert_required_keys_present!(mapping)
+      mapping.keys.each do |key|
+        raise "Missing one of #{REQUIED_KEYS}" \
+          unless REQUIED_KEYS.include? key.to_s
+      end
+    end
+
     def store_row(row_hash)
       file_path            = build_file_path(row_hash)
       normalized_row_hash  = normalize_row_hash(row_hash)
@@ -59,11 +69,13 @@ module Hakoy
     end
 
     def append_row_to_file(file_path, row_hash)
-      @append_strategy.append_row_to_file(file_path, row_hash)
+      @append_strategy.append_row_to_file file_path, row_hash
     end
 
     def finalize_store!
-      @append_strategy.finalize!(@uid_key)
+      @append_strategy.finalize! \
+        uid_key: DEFAULT_UID_KEY,
+        keys_mapping: @required_keys_mapping
     end
   end
 end

data/spec/hakoy/file_appender_spec.rb

@@ -14,11 +14,22 @@ describe Hakoy::FileAppender do
   let(:file)      { '28.csv' }
   let(:file_path) { File.join dir, file }
   let(:uid_key)   { 'order_id' }
+  let(:mapping)   do
+    {
+      customer:  'customer',
+      product:   'product',
+      timestamp: 'timestamp',
+      price:     'price',
+      quantity:  'quantity',
+      order_id:  'order_id'
+    }
+  end
 
   after { FileUtils.remove_dir(dir, true) }
 
   describe 'for csv file format' do
-    before { described_class.(file_path, row_hash) }
+
+    before { described_class.(file_path, row_hash, keys_mapping: mapping) }
 
     it 'makes directory if not exist' do
       expect(File.directory?(dir)).to be_true
@@ -30,31 +41,32 @@ describe Hakoy::FileAppender do
 
     it 'write header row to the file' do
       header_row = parse_csv_file(file_path)[0]
-      expect(header_row).to eq(row_hash.keys)
+      expect(header_row).to \
+        eq(%w(customer product timestamp price quantity order_id))
     end
 
     it 'write row to the file' do
       row = parse_csv_file(file_path)[1]
-      expect(row).to eq(row_hash.values)
+      expect(row).to eq(row_hash.values_at(*mapping.values))
     end
 
     it 'accepts string as :file_path param' do
       another_file_path = File.join(dir, '30.csv').to_s
 
-      described_class.(another_file_path, row_hash)
+      described_class.(another_file_path, row_hash, keys_mapping: mapping)
       expect(File.exists?(another_file_path)).to be_true
     end
 
     it 'appends to the file' do
       row_hash[uid_key] = '1002'
 
-      described_class.(file_path, row_hash, uid_key: uid_key)
+      described_class.(file_path, row_hash, uid_key: uid_key, keys_mapping: mapping)
       result = File.readlines(file_path).last
       expect(result).to include('1002')
     end
 
     it 'skips duplicates' do
-      described_class.(file_path, row_hash, uid_key: uid_key)
+      described_class.(file_path, row_hash, uid_key: uid_key, keys_mapping: mapping)
       expect(File.readlines(file_path).length).to eq(2)
     end
   end

data/spec/hakoy/row_normalizer_spec.rb

@@ -10,9 +10,15 @@ describe Hakoy::RowNormalizer do
 
     let(:conf) do
       {
-        uid_key: 'id',
-        required_keys: \
-          %w(order_id product customer price timestamp quantity)
+        uid_key: 'uid',
+        required_keys: [
+          'customer',
+          'product',
+          'timestamp',
+          'price',
+          'quantity',
+          'order_id'
+        ]
       }
     end
 
@@ -27,14 +33,14 @@ describe Hakoy::RowNormalizer do
 
     it 'generates a unique id based on required keys' do
       result  = @row_normalizer.normalize(input)
-      expect(result['id']).to_not be_nil
+      expect(result['uid']).to_not be_nil
 
       result2 = @row_normalizer.normalize(input)
-      expect(result['id']).to eq(result['id'])
+      expect(result2['uid']).to eq(result['uid'])
 
       input['order_id'] = '1002'
       result3 = @row_normalizer.normalize(input)
-      expect(result3['id']).to_not eq(result['id'])
+      expect(result3['uid']).to_not eq(result['uid'])
     end
 
     it 'raises error if any of the required keys not found' do

data/spec/hakoy_spec.rb

@@ -1,28 +1,35 @@
 require 'spec_helper'
 
 describe Hakoy do
+  after do
+    FileUtils.remove_dir(File.join(tmp_path, '2014'), true)
+  end
+
   it 'stores csv rows in timestamp sliced directories' do
     conf = {
       timestamp_key: 'Created at',
       db_dir: tmp_path,
       output_format: 'csv',
-      required_keys: [
-        'Billing Name',
-        'Lineitem name',
-        'Created at',
-        'Lineitem price',
-        'Lineitem quantity',
-        'Name' # order_id
-      ]
+      required_keys_mapping: {
+        customer:  'Billing Name',
+        product:   'Lineitem name',
+        timestamp: 'Created at',
+        price:     'Lineitem price',
+        quantity:  'Lineitem quantity',
+        order_id:  'Name'
+      }
     }
     Hakoy.(fixture_file('orders.csv'), conf)
 
     file1 = File.join tmp_path, '2014/5/26.csv'
     file2 = File.join tmp_path, '2014/5/28.csv'
+
     [file1, file2].each do |file|
       expect(File.exist?(file)).to be_true
     end
 
-    FileUtils.remove_dir(File.join(tmp_path, '2014'), true)
+    header = CSV.read(file1).first
+    expected_header = conf[:required_keys_mapping].keys.map(&:to_s)
+    expect(header).to match_array(expected_header)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hakoy
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Lin He
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-12 00:00:00.000000000 Z
+date: 2014-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler