url_trimmer 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 208db8066a5128f36d5d2a23435569e278346eb4
-  data.tar.gz: 353abe12c38767112e252f3d709fc78ce1a0fa2e
+  metadata.gz: 04dae9e3b37ea013c12031188024c628f8df916a
+  data.tar.gz: 32d6caa837aeb69f97ab97c2609b6b3a7ac65dc7
 SHA512:
-  metadata.gz: 626f3ad8f74050eb434382f8f40cdc2830dfac10812044db58ba99d0e9a0662eed8e78f705c25d5a32ce2b1ee6e405e6accc37e5d36df59a38f7ec71291a668c
-  data.tar.gz: b20cafc045eb1cf0cad0b8b2cb9e6fe60bb6c36de85efb51d0571959b9c41f008a1d49a044c1f087d30e7d35c5492a90646590de40c400b98f2a621c91f5a349
+  metadata.gz: b7a84541cf9f49aa591fc59ed6063faf3da67ba53cbb6d74ac97cac31b577829460c04a52e0459163096f3693b30825883e5dd4dd9aef585f89fbfb158668258
+  data.tar.gz: 1ba47225a09ddf0d16c233bc59e7680d70550b6f725a895eb4601f185569c4358862090b4b1087a94c070ad1f1a3ee85e005ca83d261f560908a2eb927a2061c

data/.gitignore

@@ -22,3 +22,4 @@ tmp
 mkmf.log
 *.swp
 .rbenv-gemsets
+*.komodoproject

data/README.md

@@ -10,7 +10,17 @@ Install it yourself as:
 
 ## Usage
 
-url-trimmer urls1.txt urls2.txt
+### URL Trimmer
+
+```bash
+url-trimmer urls1.txt urls2.txt .. urlsN.txt > unique-urls.txt
+```
+
+### URL Differ
+
+```bash
+url-differ urls1.txt urls2.txt > urls2-unique.txt
+```
 
 ## Contributing
 

data/bin/url-differ

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/url_trimmer/differ"
+
+file1, file2 = ARGV[0], ARGV[1]
+urls1 = File.readlines(file1).reject { |line| line.empty? }.map(&:chomp)
+urls2 = File.readlines(file2).reject { |line| line.empty? }.map(&:chomp)
+unique_urls = URLTrimmer::Differ.between(urls1, urls2)
+print "#{unique_urls.join("\n")}\n"

data/lib/url_trimmer/differ.rb

@@ -0,0 +1,19 @@
+require_relative "../url_trimmer"
+
+module URLTrimmer
+  class Differ
+    def self.between(urls1, urls2)
+      unique_urls1 = Worker.uniq_by_domain(urls1)
+      unique_urls1.map! { |url| url[Worker::URL_REGEXP, 0]  }
+      domains1 = unique_urls1.map { |url| DomainName(url).domain }
+      
+      unique_urls2 = Worker.uniq_by_domain(urls2)
+      unique_urls2.reject! do |url|
+        u = url[Worker::URL_REGEXP, 0]
+        domains1.include?(DomainName(u).domain)
+      end
+      unique_urls2.sort!
+      unique_urls2
+    end
+  end
+end

data/lib/url_trimmer/version.rb

@@ -1,3 +1,3 @@
 module URLTrimmer
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/spec/lib/url_trimmer/differ_spec.rb

@@ -0,0 +1,21 @@
+require File.expand_path("spec/spec_helper")
+require File.expand_path("lib/url_trimmer/differ")
+
+module URLTrimmer
+  describe Differ do
+    describe ".between(urls1, urls2)" do
+      let(:urls1) { %w(http://www.google.com/1 https://www.google.com.ar/2) }
+      let(:urls2) do
+        %w(https://www.google.com/3 http://www.google.com.ar/4 http://www.google.com.br/5 http://www.example.com/6)
+      end
+      
+      it "removes URLs from urls2 already present in urls1" do
+        unique_urls = Differ.between(urls1, urls2)
+        
+        unique_urls.size.must_equal 2
+        unique_urls.must_include("http://www.google.com.br/5")
+        unique_urls.must_include("http://www.example.com/6")
+      end
+    end
+  end
+end

data/url_trimmer.gemspec

@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest", "~> 5.3.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: url_trimmer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Cristian Rasch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-30 00:00:00.000000000 Z
+date: 2014-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: domain_name
@@ -52,10 +52,25 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.3.5
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.3.5
 description: 
 email:
 - cristian@box.cristianrasch.com.ar
 executables:
+- url-differ
 - url-trimmer
 extensions: []
 extra_rdoc_files: []
@@ -66,9 +81,12 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/url-differ
 - bin/url-trimmer
 - lib/url_trimmer.rb
+- lib/url_trimmer/differ.rb
 - lib/url_trimmer/version.rb
+- spec/lib/url_trimmer/differ_spec.rb
 - spec/lib/url_trimmer_spec.rb
 - spec/spec_helper.rb
 - url_trimmer.gemspec
@@ -98,5 +116,6 @@ specification_version: 4
 summary: Reads in plain text files with one URL per line and outputs a list of unique
   URLs by domain
 test_files:
+- spec/lib/url_trimmer/differ_spec.rb
 - spec/lib/url_trimmer_spec.rb
 - spec/spec_helper.rb