sycsvpro 0.1.10 → 0.1.11

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    sycsvpro (0.1.10)
+    sycsvpro (0.1.11)
       gli (= 2.9.0)
       timeleap (~> 0.0.1)
 

data/README.md

@@ -5,6 +5,8 @@ Processing of csv files. *sycsvpro* offers following functions
 
 * analyze a csv file
 * extract rows and columns from a file
+* remove duplicate lines from a file where duplicates are identified by key
+  columns (since version 0.1.11)
 * collect values of rows and assign them to categories
 * map column values to new values
 * allocate column values to a key column (since version 0.0.4)
@@ -76,6 +78,14 @@ Extract row 1,2 and 10-20 as well as columns 4 and 6-7
 
     $ sycsvpro -f in.csv -o out.csv extract -r 1,2,10-20 -c 4,6-7
 
+
+Unique
+------
+Remove duplicate lines from a file. Duplicates are identified by key columns.
+If no key columns are provided the whole line is checked for uniqueness
+
+    $ sycsvpro -f in.csv -o out.csv unique -r 1,2,8-12 -c 4,10-15 -k 0,1
+
 Collect
 -------
 Collect all product rows (2, 3 and 4) to the category product
@@ -473,6 +483,11 @@ Version 0.1.10
 * Fix ~/.syc/sycsvpro system directory creation when no .syc directory is 
  available
 
+Version 0.1.11
+--------------
+* Unique removes duplicate lines from the infile. Duplicate lines are identified
+  by key columns
+
 Installation
 ============
 [![Gem Version](https://badge.fury.io/rb/sycsvpro.png)](http://badge.fury.io/rb/sycsvpro)

data/lib/sycsvpro/unique.rb

@@ -0,0 +1,77 @@
+require 'set'
+
+# Operating csv files
+module Sycsvpro
+
+  # Removes copies of rows identified by key values
+  #
+  # | Name | Street | Town | Country |
+  # | ---- | ------ | ---- | ------- |
+  # | Jane | Canal  | Win  | CA      |
+  # | Jack | Long   | Van  | CA      |
+  # | Jean | Sing   | Ma   | DE      |
+  # | Jane | Canal  | Win  | CA      |
+  #
+  # Remove copies based on column 0 (Name)
+  #
+  # | Name | Street | Town | Country |
+  # | ---- | ------ | ---- | ------- |
+  # | Jane | Canal  | Win  | CA      |
+  # | Jack | Long   | Van  | CA      |
+  # | Jean | Sing   | Ma   | DE      |
+  class Unique
+
+    include Dsl
+
+    # infile contains the data that is operated on
+    attr_reader :infile
+    # outfile is the file where the result is written to
+    attr_reader :outfile
+    # filter that is used for rows
+    attr_reader :row_filter
+    # filter that is used for columns
+    attr_reader :col_filter
+
+    # Creates a new Unique
+    # :call-seq:
+    #   Sycsvpro::Unique.new(infile: "infile.csv",
+    #                        outfile: "outfile.csv",
+    #                        rows:    "1,3-4",
+    #                        cols:    "0,2,4-6",
+    #                        key:     "0,1").execute
+    def initialize(options = {})
+      @infile  = options[:infile]
+      @outfile = options[:outfile]
+      @row_filter = RowFilter.new(options[:rows], df: options[:df])
+      @col_filter = ColumnFilter.new(options[:cols], df: options[:df])
+      @key_filter = ColumnFilter.new(options[:key], df: options[:df])
+      @keys = Set.new
+    end
+
+    # Removes the duplicates from infile and writes the result to outfile
+    def execute
+      File.open(@outfile, 'w') do |out|
+        File.open(@infile, 'r').each_with_index do |line, index|
+          line = line.chomp
+
+          next if line.empty?
+          
+          line = unstring(line).chomp
+
+          extraction = col_filter.process(row_filter.process(line, row: index))
+
+          next unless extraction
+
+          key = @key_filter.process(line)
+          
+          unless @keys.include? key
+            out.puts extraction
+            @keys << key
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/sycsvpro/version.rb

@@ -1,5 +1,5 @@
 # Operating csv files
 module Sycsvpro
   # Version number of sycsvpro
-  VERSION = '0.1.10'
+  VERSION = '0.1.11'
 end

data/spec/sycsvpro/unique_spec.rb

@@ -0,0 +1,42 @@
+require 'sycsvpro/unique'
+
+module Sycsvpro
+
+  describe Unique do
+
+    before do
+      @infile  = File.join(File.dirname(__FILE__), "files/customer-address.csv")
+      @outfile = File.join(File.dirname(__FILE__), "files/out.csv")
+    end
+    
+    it "should remove copies" do
+
+      rows = "0-10"
+      cols = "0,1-3"
+      key  = "0,1"
+
+      Sycsvpro::Unique.new(infile:  @infile,
+                           outfile: @outfile,
+                           rows:    rows,
+                           cols:    cols,
+                           key:     key).execute
+
+      result = [ "Name;Street;Town;Country",
+                 "Jane;Canal;Vancouver;CA",
+                 "John;Milton;Washington;US",
+                 "Jne;Canal;Vancouver;CA",
+                 "Jhn;Milton;Washington DC;US" ]
+
+      rows = 0
+
+      File.open(@outfile).each_with_index do |line, index|
+        line.chomp.should eq result[index]
+        rows += 1
+      end
+
+      rows.should eq result.count
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sycsvpro
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.1.11
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-05 00:00:00.000000000 Z
+date: 2014-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -151,6 +151,7 @@ files:
 - lib/sycsvpro/script_list.rb
 - lib/sycsvpro/sorter.rb
 - lib/sycsvpro/table.rb
+- lib/sycsvpro/unique.rb
 - lib/sycsvpro/version.rb
 - spec/sycsvpro/aggregator_spec.rb
 - spec/sycsvpro/allocator_spec.rb
@@ -174,6 +175,7 @@ files:
 - spec/sycsvpro/script_list_spec.rb
 - spec/sycsvpro/sorter_spec.rb
 - spec/sycsvpro/table_spec.rb
+- spec/sycsvpro/unique_spec.rb
 - sycsvpro.gemspec
 - sycsvpro.rdoc
 homepage: https://github.com/sugaryourcoffee/syc-svpro