deriving_license 0.2.0 → 0.2.1

data/README.md

@@ -6,69 +6,89 @@ Finds the license agreements for all gems in your Gemfile. This is achieved by r
 Strategies:
 * from\_gem\_specification
 * from\_license\_file (not yet implemented)
-* from\_scraping\_homepage (not yet implemented)
+* from\_scraping\_homepage
 * from\_parsing\_readme (not yet implemented)
 
 Example output:
 
-    $ deriving_license ~/Code/rails-sample-app/Gemfile
+    $ deriving_license ../bc/A-A/Gemfile
     Determining license for rails:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for adt:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for app_constants:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for bcrypt-ruby:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for bootstrap-sass:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for jquery-rails:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for json:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for nokogiri:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for pg:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for quiet_assets:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for rack-protection:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for symmetric-encryption:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for newrelic_rpm:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for uglifier:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for sass-rails:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for coffee-rails:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for rspec-rails:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for capybara:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for factory_girl_rails:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for faker:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for better_errors:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for binding_of_caller:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for debugger:
     	Trying from_gem_specification strategy...SUCCESS
     Determining license for simplecov:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for ci_reporter:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     Determining license for sqlite3:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...FAILED
     Determining license for mysql2:
     	Trying from_gem_specification strategy...FAILED
+    	Trying from_scraping_homepage strategy...SUCCESS
     
     Detected 4 known licenses:
-    MIT: Expat License (4 instances)[http://directory.fsf.org/wiki/License:Expat]
-    Ruby: Ruby License (2 instances)[http://www.ruby-lang.org/en/about/license.txt]
+    MIT: Expat License (12 instances)[http://directory.fsf.org/wiki/License:Expat]
+    Ruby: Ruby License (6 instances)[http://www.ruby-lang.org/en/about/license.txt]
     BSD: FreeBSD Copyright (2 instances)[http://www.freebsd.org/copyright/freebsd-license.html]
-    GPL: GNU General Public License (1 instance)[http://en.wikipedia.org/wiki/GNU_General_Public_License]
+    GPL: GNU General Public License (2 instances)[http://en.wikipedia.org/wiki/GNU_General_Public_License]

data/deriving_license.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name        = 'deriving_license'
-  s.version     = '0.2.0'
+  s.version     = '0.2.1'
   s.summary     = "Deriving Licence finds the license agreements for all gems in your Gemfile"
   s.description = "Deriving Licence finds the license agreements for all gems in your Gemfile if included in your project, or in a Gemfile passed to the included binary"
   s.authors     = ["Tom Allen"]
@@ -15,4 +15,5 @@ Gem::Specification.new do |s|
   
   s.add_runtime_dependency "gemnasium-parser"
   s.add_runtime_dependency "safe_yaml"
+  s.add_runtime_dependency "curb"
 end

data/lib/deriving_license.rb

@@ -1,7 +1,7 @@
 require "gemnasium/parser"
 require "bundler"
 require "safe_yaml"
-require "open-uri"
+require "curb"
 
 class DerivingLicense
   
@@ -33,6 +33,8 @@ class DerivingLicense
     "from_gem_specification",
     "from_scraping_homepage"
   ]
+  
+  @@specs_cache = {} # Cache of gem specifications previously fetched.
 
   def self.run(path=nil)
     unless path
@@ -97,12 +99,16 @@ class DerivingLicense
   end
   
   def self.get_gem_spec(dep)
+    # Check spec cache first.
+    @spec = @@specs_cache[dep]
+    return @spec if @spec
     # See if the gem is installed locally, and if not add -r to call
     Bundler.with_clean_env do # This gets out of the bundler context.
       remote = /#{dep}/.match( `gem list #{dep}` ) ? "" : "-r "      
       yaml = `gem specification #{remote}#{dep} --yaml`
       @spec = YAML.load(yaml, :safe => true)
     end
+    @@specs_cache[dep] = @spec # Cache it.
     @spec
   end
   
@@ -121,11 +127,15 @@ class DerivingLicense
   def self.from_scraping_homepage(dep)
     spec = get_gem_spec(dep)
     licenses = []
-    unless spec["homepage"]
-      []
+    unless spec["homepage"] and !spec["homepage"].empty?
+      return []
+    end
+    begin
+      content = Curl::Easy.perform(spec["homepage"]){|easy| easy.follow_location = true; easy.max_redirects=nil}.body_str
+    rescue
+      return []
     end
-    content = open(spec["homepage"])
-    content.each_line do |l|
+    content.split('\n').each do |l|
       if /license/.match(l)
         # Found the word "license", so now look for known license names.
         (@@license_details.keys + @@license_aliases.keys).each do |n|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deriving_license
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Deriving Licence finds the license agreements for all gems in your Gemfile
   if included in your project, or in a Gemfile passed to the included binary
 email: tom@jugglethis.net