common-crawl-index 0.0.2.alpha → 0.0.2

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## v0.0.2
+
+* Added runtime dependancy for aws-sdk and addressable gems instead of placing them in Gemfile
+
 ## v0.0.1.alpha
 
 * initial release

data/Gemfile

@@ -1,7 +1,4 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in cc-url-index.gemspec
-gemspec
-
-gem 'aws-sdk'
-#gem 'open3'
+gemspec

data/README.md

@@ -1,6 +1,6 @@
 # CommonCrawlIndex
 
-TODO: Write a gem description
+Use this gem to access [Common Crawl URL Index](http://commoncrawl.org/common-crawl-url-index/) with ruby. You can get more information on format and original python implementation at https://github.com/trivio/common_crawl_index
 
 ## Installation
 
@@ -18,7 +18,33 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+When using with Rails in `config/initializers/common_crawl_index.rb`
+
+```ruby
+CommonCrawlIndex::Client.config({
+  :access_key_id =>  "amazon aws access_key",
+  :secret_access_key => "amazon aws secret_key",
+  :cc_index_path => "s3://aws-publicdatasets/common-crawl/projects/url-index/url-index.1356128792" # optional
+})
+```
+
+And to find URLs matching certain prefix use following syntax
+
+```ruby
+client = CommonCrawlIndex::Client.new(AMAZON_ACCESS_KEY_ID, AMAZON_SECRET_ACCESS_KEY)
+
+# or
+
+client = CommonCrawlIndex::Client.new() # already configured
+
+url = "http://www.amazon.com/"
+
+client.find_by_prefix(url) do |url_data|
+  # get all URLs starting with http://www.amazon.com/
+end
+```
+
+See `spec/basic_spec.rb` for more examples on usage.
 
 ## Contributing
 

data/common-crawl-index.gemspec

@@ -18,4 +18,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency('aws-sdk')
+  gem.add_runtime_dependency('addressable')
 end

data/lib/common-crawl-index.rb

@@ -1,7 +1,7 @@
-require "common-crawl-index/version"
+require 'common-crawl-index/version'
 require 'aws-sdk'
 require 'open3'
-
+require 'addressable/uri'
 
 module CommonCrawlIndex
   class Client
@@ -42,7 +42,7 @@ module CommonCrawlIndex
 
     def self.normalize_url(url, append_scheme = true)
       url_to_find = url
-      norm_url_to_find = URI(url_to_find)
+      norm_url_to_find = Addressable::URI.parse(url_to_find)
       norm_url_to_find.host = norm_url_to_find.host.split(".").reverse.join(".")
       norm_url = norm_url_to_find.to_s
       norm_url = norm_url[norm_url.index("\/\/")+2..-1]
@@ -58,7 +58,7 @@ module CommonCrawlIndex
         scheme = normalized_url[colon_index+1..-1] if colon_index
       end
       url_with_scheme = scheme + "://" + normalized_url[0..colon_index-1]
-      uri = URI(url_with_scheme)
+      uri = Addressable::URI.parse(url_with_scheme)
       uri.host = uri.host.split(".").reverse.join(".")
       uri.to_s
     end
@@ -119,6 +119,7 @@ module CommonCrawlIndex
         end
         cur_loc = nil_loc + 32 + 1
       end
+      true
     end
 
     def read(target_range)

data/lib/common-crawl-index/version.rb

@@ -1,3 +1,3 @@
 module CommonCrawlIndex
-  VERSION = "0.0.2.alpha"
+  VERSION = "0.0.2"
 end

data/spec/basic_spec.rb

@@ -26,7 +26,7 @@ describe CommonCrawlIndex do
   it "should find by prefix" do
     client = CommonCrawlIndex::Client.new(AMAZON_ACCESS_KEY_ID, AMAZON_SECRET_ACCESS_KEY)
 
-    total_urls_to_test = 100
+    total_urls_to_test = 2500
 
     url = "http://www.amazon.com/"
     normalized_url = CommonCrawlIndex::Client.normalize_url(url, false)
@@ -52,6 +52,9 @@ describe CommonCrawlIndex do
   it "should normalize the urls correctly" do
     normalized_url =  CommonCrawlIndex::Client.normalize_url("http://www.google.com/test/path")
     normalized_url.should == "com.google.www/test/path:http"
+
+    normalized_url =  CommonCrawlIndex::Client.normalize_url("http://www.google.com/cse?cx=009462381166450434430:-woy8fnynf8&ie=UTF-8&q=physician+Cardiology+Diagnostics+md+\"Greater+Atlanta+Area\"+-recruiter&sa=Search&siteurl=www.google.com/cse/home%3Fcx%3D009462381166450434430%253A-woy8fnynf8")
+    normalized_url.should == "com.google.www/cse?cx=009462381166450434430:-woy8fnynf8&ie=UTF-8&q=physician+Cardiology+Diagnostics+md+\"Greater+Atlanta+Area\"+-recruiter&sa=Search&siteurl=www.google.com/cse/home%3Fcx%3D009462381166450434430%253A-woy8fnynf8:http"
   end
 
   it "should normalize the urls correctly without scheme" do
@@ -62,6 +65,9 @@ describe CommonCrawlIndex do
   it "should denormalize the urls correctly" do
     url =  CommonCrawlIndex::Client.denormalize_url("com.google.www/test/path:http")
     url.should == "http://www.google.com/test/path"
+
+    url =  CommonCrawlIndex::Client.denormalize_url("com.google.www/cse?cx=009462381166450434430:-woy8fnynf8&ie=UTF-8&q=physician+Cardiology+Diagnostics+md+\"Greater+Atlanta+Area\"+-recruiter&sa=Search&siteurl=www.google.com/cse/home%3Fcx%3D009462381166450434430%253A-woy8fnynf8:http")
+    url.should == "http://www.google.com/cse?cx=009462381166450434430:-woy8fnynf8&ie=UTF-8&q=physician+Cardiology+Diagnostics+md+\"Greater+Atlanta+Area\"+-recruiter&sa=Search&siteurl=www.google.com/cse/home%3Fcx%3D009462381166450434430%253A-woy8fnynf8"
   end
 
   it "should denormalize the urls correctly without scheme" do

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: common-crawl-index
 version: !ruby/object:Gem::Version
-  version: 0.0.2.alpha
-  prerelease: 6
+  version: 0.0.2
+  prerelease: 
 platform: ruby
 authors:
 - Amit Ambardekar
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-23 00:00:00.000000000 Z
+date: 2013-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -27,6 +27,38 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Access coomon crawl URL index
 email:
 - amitamb@gmail.com
@@ -61,9 +93,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24