slicing 0.1.0.pre → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac5e311db2eb6d863920974446dabd370e217a71
-  data.tar.gz: 5a6b337cc80ad664fdd23a41af0b6a406755f203
+  metadata.gz: 014cfbd59f2b2e16e160d0188b59984f949eb0f6
+  data.tar.gz: b42b46be39904cbf248cfea88bc1b8eee2e0ee1a
 SHA512:
-  metadata.gz: b45eb3134ff3c96fbe7da6f76ccb5b987abb7caab0d55e6cf3cc972e2474132f14559d9fa41111df5557ced0952dad12e9cddd4f55c9fd6dead6f8958c35b5c4
-  data.tar.gz: 386af6723a8e33bacff56a6e45f31b57316c79c03a815d4533c73e51b6c2f31323c62dc1b1cae5ac18916018834181a37ebd7a38826282946895298a3b6b9e20
+  metadata.gz: 49dcfb2d5407fc30018d660bfaa07ea56c82a6d848a5b8de585450d4555ba16415c393cc9cfa01537801ec2a0eae3b48b024d38803799e14e2616579fdf985de
+  data.tar.gz: a7717fc4df45630346a4c9e6003ae34f13617ec0d542631bc61e04c0a7d6caed253182d1ff61bb560b2ec4ddc2dfefd9c873aafb5d5ec28295fef7e84be5c9bb

data/README.md

@@ -1,5 +1,14 @@
 # Slicing
-:v: Slice and dice your csv files for further analysis
+:v: Slice and dice your csv files for further analysis.
+
+# Reason for making slicing
+No database required, mom!
+
+Data slicing and cleaning usually happen within a database like mysql or postgresql. And you will need to use sql command to manipulate the data.
+
+I thought why can't I slice and dice them on the go. And take a subset of the data and peek it through excel or a text-editor.
+
+Hence, slicing is created to make it easy to slice csv files from terminal.
 
 ## Installation
 
@@ -18,16 +27,17 @@ Or install it yourself as:
     $ gem install slicing
 
 ## Usage
+`slicing mask` - mask the column with md5.
 
-`slicing subset` -
+`slicing subset` - return a subset of 10 lines of the bigger csv file.
 
-`slicing head` -
+`slicing head` - return the header of the csv file.
 
-`slicing rm` -
+`slicing rm` - remove the column from csv file.
 
-`slicing first` -
+`slicing first` - return the first line of data of the csv file.
 
-`slicing count` -
+`slicing count` - return the total row and column of the csv file.
 
 ## Development
 

data/lib/slicing.rb

@@ -1,4 +1,5 @@
 require "slicing/version"
+require 'digest/md5'
 require 'thor'
 require 'csv'
 
@@ -6,23 +7,72 @@ module Slicing
   class Base < Thor
     check_unknown_options!
     package_name 'slicing'
-    default_task :hello
+    default_task :help
 
-    desc :rm, ""
+    desc :sample, "create a sample output"
+    def sample path, output_path, size
+      file_csv = CSV.read(path,:headers=> true, :encoding => "ISO8859-1:utf-8")
+      sample = file_csv.sample(size)
+      CSV.open(output_path, "a+") do |csv|
+        sample.each do |value|
+          csv << value
+        end
+      end
+    end
+
+    desc :freq, "calculate item frequencies"
+    def freq path, column_name, output_path
+      file_to_count = "./#{path}.csv"
+      output = "./#{path}-counted.csv"
+      file_to_count_csv = CSV.read(file_to_count,:headers=> true, :encoding => "ISO8859-1:utf-8")
+      unique_nric_array = file_to_count_csv[column_name]
+      unique_nric = []
+      unique_nric_array.each_with_index do |value, index|
+        unique_nric.push(value) if index !=0
+      end
+
+      final_hash = score(unique_nric)
+      CSV.open(output, "a+") do |csv|
+        final_hash.each do |value|
+          csv << [value[0], value[1]]
+        end
+      end
+    end
+
+
+    desc :mask, "mask a particular column"
+    def mask path, column_name, output_path
+      original = CSV.read(path, { headers: true, return_headers: true, :encoding => "ISO8859-1:utf-8"})
+      CSV.open(output_path, 'a+') do |csv|
+        original.each do |row|
+          csv << array
+        end
+      end
+    end
+
+    desc :rm, "remove a column"
+    method_option :utf, type: :string, aliases: '-u', default: "ISO8859-1:utf-8"
+    method_option :headers, type: :boolean, aliases: '-h', default: true
+    method_option :rowsep, type: :string, aliases: '-r', default: nil
     def rm path, column_name, output
-      data = CSV.read(path, :headers=> false, :encoding => "ISO8859-1:utf-8") #2014
-      data.delete(column_name)
-      CSV.open(output,"a+") do |csv|
-        data.each_with_index do |row,index|
+      # headers, rowsep, utf = process_options(options[:headers], options[:rowsep], options[:utf])
+      if options[:rowsep] != nil
+        original = CSV.read(path, { headers: options[:headers], return_headers: options[:headers], :row_sep=> options[:rowsep], :encoding => options[:utf]})
+      else
+        original = CSV.read(path, { headers: options[:headers], return_headers: options[:headers], :encoding => options[:utf]})
+      end
+      original.delete(column_name)
+      CSV.open(output, 'a+') do |csv|
+        original.each do |row|
           csv << row
         end
       end
     end
 
-
-    desc :first, ""
+    desc :first, "display the first numbers of line"
+    method_option :line, type: :numeric, aliases: '-l', default: 100
     def first csv_file #, value=100
-      stop = 100
+      stop = options[:line]
       counter = 0
       CSV.foreach(csv_file, :headers => false, encoding: "ISO8859-1:utf-8") do |row|
         exit if counter == stop
@@ -31,11 +81,10 @@ module Slicing
           puts row
         rescue
         end
-
       end
     end
 
-    desc :head, ""
+    desc :head, "show the headers"
     def head csv_file
       CSV.foreach(csv_file, :headers => false, encoding: "ISO8859-1:utf-8") do |row|
         puts row
@@ -45,18 +94,29 @@ module Slicing
       end
     end
 
+    desc :unique, "calculate number of unique values in column"
+    def unique path, column_name
+      data = CSV.read(path, :headers => true, return_headers: true, encoding: "ISO8859-1:utf-8")
+      array = data[column_name]
+      puts array.uniq.count if array != nil
+    end
+
 
-    desc :count, ""
+    desc :count, "count the number of rows and columns"
     def count csv_file
-      data = CSV.read(csv_file)
-      puts "#{data.count} rows"
+      data = CSV.read(csv_file, :headers => false, encoding: "ISO8859-1:utf-8")
+      puts "#{data.count} rows #{data[0].count} columns"
+      puts "---"
+      puts "#{data[0]}"
     end
 
-    desc :subset, ""
-    def subset csv_file, output, value=10
+    desc :subset, "create a subset of the data"
+    method_option :line, type: :numeric, aliases: '-l', default: 1000
+    def subset(csv_file, output)
       path = csv_file
       output_directory =  output #"/Users/ytbryan/Desktop/output/subset-2015.csv" #output directory
-      stop = value
+      # options[:num] == nil ? (stop = 10) : (stop = options[:num])
+      stop = options[:line]
       counter = 0
       CSV.foreach(path, :headers => false, encoding: "ISO8859-1:utf-8") do |row|
         exit if counter == stop
@@ -70,5 +130,45 @@ module Slicing
       end
     end
 
+    # desc :subsetagain, ""
+    # def subsetagain csv_file, output, value=10
+    #   path = csv_file
+    #   output_directory =  output #"/Users/ytbryan/Desktop/output/subset-2015.csv" #output directory
+    #   stop = value
+    #   counter = 0
+    #   CSV.foreach(path, :headers => false, :row_sep => "\r\n", encoding: "ISO8859-1:utf-8") do |row|
+    #     exit if counter == stop
+    #     begin
+    #       counter = counter + 1
+    #       CSV.open(output_directory, "a+") do |csv|
+    #         csv << row
+    #       end
+    #     rescue
+    #     end
+    #   end
+    # end
+
+    private
+
+    def process_options headers, rowsep, utf
+      if headers == nil
+        headers = true
+      else
+        headers = headers
+      end
+      return true, "\r\n" , "ISO8859-1:utf-8"
+    end
+
+    def masking(value)
+      value != nil ? answer = Digest::MD5.hexdigest(value) : answer
+    end
+
+    def score( array )
+      hash = Hash.new(0)
+      array.each{|key| hash[key] += 1}
+      hash
+    end
+
+
   end
 end

data/lib/slicing/version.rb

@@ -1,3 +1,3 @@
 module Slicing
-  VERSION = "0.1.0.pre"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: slicing
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre
+  version: 0.1.0
 platform: ruby
 authors:
 - Bryan Lim
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,7 +58,6 @@ files:
 - bin/slicing
 - lib/slicing.rb
 - lib/slicing/version.rb
-- slicing-0.1.0.gem
 - slicing.gemspec
 homepage: http://github.com/ytbryan/slicing
 licenses:
@@ -75,9 +74,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1