license_matcher 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75f4cc4338e87a4244875d5172770867b63588cb
-  data.tar.gz: 3bb514e6e029140949ed3c78c1b32a1a04333a04
+  metadata.gz: bae726cf67cb9e615291d1da18e87f120fdb7ea6
+  data.tar.gz: d104b7310dc786821c56df1b3d984e9069dc1434
 SHA512:
-  metadata.gz: 1b99befeaf51c94e2f85acdc991abd02486de41ea3f7506a2e7a6402f08e4c0765b322cb480db98703da01ef286bf9751bc6fc458356e1c93c4142f611a26457
-  data.tar.gz: 67b3d8e0265f91ebd9d1818983dfea906f543dffcc08d41d3cb50aeb66d56fe76714409085e741513a36229cbaefef388e0ffe691e20e06ec558796eef6f4ac0
+  metadata.gz: 72c13e2a98ce021d8491063244f00c9ef56ec3ef7332d77448485d93d92cdcead16e20cb5dc0b01726970defed06dafa4bc2ac121a867df1393dace608a3e951
+  data.tar.gz: d11e26305040d796f072294f6bdb9c20e09e3fcaf25e2d519e0737bf1647a4ac65166795a030a35f4938b8fffbbf2cb66ae00f64edc9066b124f49aa93c090ed

data/Gemfile.lock

@@ -1,11 +1,13 @@
 PATH
   remote: .
   specs:
-    license_matcher (0.2.0.pre.alpha)
-      helix_runtime (~> 0.6.0)
+    license_matcher (0.3.0)
+      helix_runtime (~> 0.6)
+      msgpack (~> 1.1)
       narray (~> 0.6.1.2)
-      nokogiri (~> 1.8.0)
-      tf-idf-similarity (~> 0.1.6)
+      nokogiri (~> 1.8)
+      os (~> 1.0)
+      tf-idf-similarity (~> 0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -16,11 +18,12 @@ GEM
       rake (>= 10.0)
       thor (~> 0.19.4)
       toml (~> 0.1.2)
-    mini_portile2 (2.2.0)
+    mini_portile2 (2.3.0)
     msgpack (1.1.0)
     narray (0.6.1.2)
-    nokogiri (1.8.0)
-      mini_portile2 (~> 2.2.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    os (1.0.0)
     parslet (1.5.0)
       blankslate (~> 2.0)
     rake (10.5.0)
@@ -50,7 +53,6 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.15)
   license_matcher!
-  msgpack (~> 1.1.0)
   rake (~> 10.0)
   rspec (~> 3.4)
 

data/README.md

@@ -44,7 +44,7 @@ LicenseMatcher::IndexBuilder.build_index( "data/licenses", "data/index.msgpack")
 txt = File.read("fixtures/files/mit.txt");
 
 lm = LicenseMatcher::TFRubyMatcher.new("data/index.msgpack")
-m  = lm.match_text(txt, 0.9) 
+m  = lm.match_text(txt, 0.9)
 p "spdx id: #{m.get_label()}, confidence: #{m.get_score()}"
 
 ```
@@ -78,7 +78,7 @@ lm.match_rules "It is license under Apache 2.0 License."
 lm = LicenseMatcher::TFRubyMatcher.new
 
 txt = File.read "fixtures/files/mit.html"
-clean_txt = lm.preprocess_html txt # NB! it may help to increase accuracy
+clean_txt = LicenseMatcher::Preprocess.preprocess_html txt # NB! it may help to increase accuracy
 lm.match_txt clean_txt
 ```
 
@@ -95,7 +95,7 @@ lm2.match_text txt
 
 ```
 lm3 = File.read "fixtures/files/mit.txt"
-lm3.match_text txt 
+lm3.match_text txt
 ```
 
 ## Benchmarks
@@ -138,7 +138,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/fosslim/license_matcher. 
+Bug reports and pull requests are welcome on GitHub at https://github.com/fosslim/license_matcher.
 
 This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 

data/lib/license_matcher/preprocess.rb

@@ -1,75 +1,77 @@
 require 'nokogiri'
 
-module Preprocess
-  def preprocess_text(text)
-    text = safe_encode(text)
+module LicenseMatcher
+  class Preprocess
+    def self.preprocess_text(text)
+      text = safe_encode(text)
 
-    #remove markdown url tags
-    text = text.gsub(/\[.+?\]\(.+?\)/, ' ')
+      #remove markdown url tags
+      text = text.gsub(/\[.+?\]\(.+?\)/, ' ')
 
-    #remove spam words
-    text.gsub!(/\bTHE\b/i, '')
+      #remove spam words
+      text.gsub!(/\bTHE\b/i, '')
 
-    #remove some XML grabage
-    text = text.gsub(/\<\!\[CDATA.*?\]\]\>/, ' ').to_s
-    text = text.gsub(/\<\!--.+?--\>/,  ' ').to_s
-    text = text.gsub(/<\!\[CDATA.+?\]>/, ' ').to_s
+      #remove some XML grabage
+      text = text.gsub(/\<\!\[CDATA.*?\]\]\>/, ' ').to_s
+      text = text.gsub(/\<\!--.+?--\>/,  ' ').to_s
+      text = text.gsub(/<\!\[CDATA.+?\]>/, ' ').to_s
 
-    return text.to_s.strip.gsub(/\s+/, ' ')
-  end
+      return text.to_s.strip.gsub(/\s+/, ' ')
+    end
 
-  def preprocess_html(html_text)
-    # if text is HTML doc, then
-    # extract text only from visible html tags
-    text = ""
+    def self.preprocess_html(html_text)
+      # if text is HTML doc, then
+      # extract text only from visible html tags
+      text = ""
 
-    html_doc = parse_html(html_text)
-    if html_doc
-      text = clean_html(html_doc)
-    else
-      p "match_html: failed to parse html document\n#{html_text}"
+      html_doc = parse_html(html_text)
+      if html_doc
+        text = clean_html(html_doc)
+      else
+        p "match_html: failed to parse html document\n#{html_text}"
+      end
+
+      return text
     end
 
-    return text
-  end
+    def self.clean_html(html_doc)
+      body_text = ""
+      body_elements = html_doc.xpath(
+        '//p | //h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //em | //strong | //b | //td | //pre
+        | //li[not(@id) and not(@class) and not(a)] | //section//section[@class="project-info"]
+        | //blockquote | //textarea'
+      ).to_a
 
-  def clean_html(html_doc)
-    body_text = ""
-    body_elements = html_doc.xpath(
-      '//p | //h1 | //h2 | //h3 | //h4 | //h5 | //h6 | //em | //strong | //b | //td | //pre
-      | //li[not(@id) and not(@class) and not(a)] | //section//section[@class="project-info"]
-      | //blockquote | //textarea'
-    ).to_a
+      #extract text from html tag and separate them by space
+      body_elements.each {|el| body_text += ' ' + el.text.to_s}
 
-    #extract text from html tag and separate them by space
-    body_elements.each {|el| body_text += ' ' + el.text.to_s}
+      #REMOVE XML CDATA like opensource.org pages has
+      body_text = body_text.to_s.strip
+      body_text.gsub!(/\<\!\[CDATA.+?\]\]\>/i, ' ')
 
-    #REMOVE XML CDATA like opensource.org pages has
-    body_text = body_text.to_s.strip
-    body_text.gsub!(/\<\!\[CDATA.+?\]\]\>/i, ' ')
+      if body_text.empty?
+        p "match_html: document didnt pass noise filter, will use whole body content"
+        body_text = html_doc.xpath('//body').text.to_s.strip
+      end
 
-    if body_text.empty?
-      p "match_html: document didnt pass noise filter, will use whole body content"
-      body_text = html_doc.xpath('//body').text.to_s.strip
+      return body_text
     end
 
-    return body_text
-  end
+    def self.parse_html(html_text)
+      begin
+        return Nokogiri.HTML(safe_encode(html_text))
+      rescue Exception => e
+        p "failed to parse html doc: \n #{html_text} - #{e.message}"
+        return nil
+      end
+    end
 
-  def parse_html(html_text)
-    begin
-      return Nokogiri.HTML(safe_encode(html_text))
-    rescue Exception => e
-      log.error "failed to parse html doc: \n #{html_text}"
-      return nil
+    def self.safe_encode(txt)
+      txt.to_s.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+    rescue
+      p "Failed to encode text:\n #{txt}i"
+      return ""
     end
-  end
 
-  def safe_encode(txt)
-    txt.to_s.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
-  rescue
-    p "Failed to encode text:\n #{txt}i"
-    return ""
   end
-
-end
+end

data/lib/license_matcher/rule_matcher.rb

@@ -1,14 +1,11 @@
 require 'json'
 
 module LicenseMatcher
-  
-  class RuleMatcher
-    include Preprocess
 
+  class RuleMatcher
     attr_reader :licenses, :rules, :id_spdx_idx
 
-    DEFAULT_LICENSE_JSON      = 'data/spdx_licenses/licenses.json'
-
+    DEFAULT_LICENSE_JSON      = 'data/licenses.json'
 
     def initialize(license_json_file = DEFAULT_LICENSE_JSON)
 
@@ -31,6 +28,16 @@ module LicenseMatcher
       idx
     end
 
+    def match_text(text, min_confidence = 0.0)
+      res = match_rules(text, true)
+      if res.empty?
+        Match.new("", 0.0)
+      else
+        spdx_id, score = res.first
+        Match.new(spdx_id, score.to_f)
+      end
+    end
+
     # finds matching regex rules in the text and sorts matches by length of match
     # ps: not very efficient, but good enough to handle special cases;
     # @args:
@@ -39,7 +46,7 @@ module LicenseMatcher
     #   [[spdx_id, score, matching_rule, matching_length],...]
     def match_rules(text, early_exit = false)
       matches = []
-      text = preprocess_text(text)
+      text = LicenseMatcher::Preprocess.preprocess_text(text)
 
       #if text is already spdx_id, then shortcut matching
       if @rules.has_key?(text.downcase)
@@ -90,7 +97,6 @@ module LicenseMatcher
     def read_json_file(file_path)
       JSON.parse(File.read(file_path), {symbolize_names: true})
     rescue
-      log.info "Failed to read json file `#{file_path}`"
       nil
     end
 
@@ -144,7 +150,7 @@ module LicenseMatcher
       end
 
 
-      spdx_name = preprocess_text(spdx_item[:name])
+      spdx_name = LicenseMatcher::Preprocess.preprocess_text(spdx_item[:name])
       spdx_name.gsub!(/\(.+?\)/, '')       #remove SPDX ids in the license names
       spdx_name.gsub!(/\./, '\\.')         #mark version dots as not regex selector
       spdx_name.gsub!(/[\*|\?|\+]/, '.')   #replace regex selector with whatever mark ~> WTFPL name
@@ -598,4 +604,4 @@ module LicenseMatcher
     end
 
   end
-end
+end

data/lib/license_matcher/tf_ruby_matcher.rb

@@ -5,7 +5,6 @@ require 'msgpack'
 module LicenseMatcher
 
   class TFRubyMatcher
-    include Preprocess
 
     attr_reader :corpus, :model, :spdx_ids
 
@@ -26,10 +25,9 @@ module LicenseMatcher
     # matches given text with SPDX licenses and returns Match object
     # returns:
     #   match - Match {label: String, score: float}
-    def match_text(text, min_confidence = DEFAULT_MIN_CONFIDENCE, is_processed_text = false)
+    def match_text(text, min_confidence = DEFAULT_MIN_CONFIDENCE)
       return [] if text.to_s.empty?
 
-      text = preprocess_text(text) if is_processed_text == false
       test_doc   = TfIdfSimilarity::Document.new(text, {:id => "test"})
 
       mat1 = @model.instance_variable_get(:@matrix)
@@ -52,7 +50,8 @@ module LicenseMatcher
     end
 
     def match_html(html_text, min_confidence = DEFAULT_MIN_CONFIDENCE)
-      match_text(preprocess_html(html_text), min_confidence)
+      clean_text = LicenseMatcher::Preprocess.preprocess_html(html_text)
+      match_text(clean_text, min_confidence)
     end
 
   #-- helpers
@@ -89,7 +88,7 @@ module LicenseMatcher
 
       idx[A_DOC_ROW].to_a.each do |doc_row|
         _, spdx_id, content, _ = doc_row
-        txt = preprocess_text content
+        txt = LicenseMatcher::Preprocess.preprocess_text content
         if txt
           spdx_ids << spdx_id
           docs << TfIdfSimilarity::Document.new(txt, :id => spdx_id)

data/lib/license_matcher/url_matcher.rb

@@ -1,10 +1,9 @@
 
 module LicenseMatcher
-
   class UrlMatcher
     attr_reader :url_index
 
-    DEFAULT_LICENSE_JSON = 'data/spdx_licenses/licenses.json'
+    DEFAULT_LICENSE_JSON = 'data/licenses.json'
 
     def initialize(license_json_file = DEFAULT_LICENSE_JSON)
       licenses_json_doc = read_json_file license_json_file
@@ -13,33 +12,42 @@ module LicenseMatcher
       @url_index =  read_license_url_index(licenses_json_doc)
     end
 
-  # Matches License.url with urls in Licenses.json and returns tuple [spdx_id, score]
+    def match_text(url_txt, min_confidence = 0.0)
+      spdx_id, score = match_url
+      if spdx_idx
+        Match.new(spdx_id, score.to_f)
+      else
+        Match.new("", 0.0)
+      end
+    end
+
+    # Matches License.url with urls in Licenses.json and returns tuple [spdx_id, score]
     def match_url(the_url)
       the_url = the_url.to_s.strip
       spdx_id = nil
 
       case the_url
-      when 'http://jquery.org/license'
+      when /jquery\.org\/license/i
         return ['mit', 1.0] #Jquery license page doesnt include any license text
-      when 'https://www.mozilla.org/en-US/MPL/'
+      when /mozilla\.org\/en-US\/MPL/i
         return ['mpl-2.0', 1.0]
-      when 'http://fairlicense.org'
+      when /fairlicense\.org/i
         return ['fair', 1.0]
-      when 'http://www.aforgenet.com/framework/license.html'
+      when /aforgenet\.com\/framework\/license/i
         return ['lgpl-3.0', 1.0]
-      when 'http://www.apache.org/licenses/'
+      when /apache\.org\/licenses/i
         return ['apache-2.0', 1.0]
-      when 'http://aws.amazon.com/apache2.0/'
+      when /aws\.amazon\.com\/apache2\.0/i
         return ['apache-2.0', 1.0]
-      when 'http://aws.amazon.com/asl/'
+      when /aws\.amazon\.com\/asl/i
         return ['amazon', 1.0]
-      when 'https://choosealicense.com/no-license/'
+      when /choosealicense\.com\/no-license/i
         return ['no-license', 1.0]
-      when 'http://www.gzip.org/zlib/zlib_license.html'
+      when /gzip\.org\/zlib\/zlib?license/i
         return ['zlib', 1.0]
-      when 'http://zlib.net/zlib-license.html'
+      when /zlib\.net\/zlib?license/i
         return ['zlib', 1.0]
-      when 'http://www.wtfpl.net/about/'
+      when /wtfpl\.net\/about/i
         return ['wtfpl', 1.0]
       end
 

data/lib/license_matcher.rb

@@ -1,9 +1,21 @@
+require 'os'
 require "helix_runtime"
 
-begin
+
+if OS.mac?
+  begin
     require "license_matcher/native"
-rescue LoadError
-    warn "Unable to load license_matcher/native. Please run `rake build`"
+  rescue LoadError
+    warn "Failed to load native extensions for OSx. Please run `rake build`"
+  end
+elsif OS.posix?
+  begin
+    require "license_matcher/native.so"
+  rescue LoadError
+    warn "Failed to load native extensions for Posix. Please run `rake build`"
+  end
+else
+  warn "Unsupported platform - you are not able to use some models;"
 end
 
 require 'license_matcher/preprocess'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: license_matcher
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Timo Sulg
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-27 00:00:00.000000000 Z
+date: 2017-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: helix_runtime
@@ -17,14 +17,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.0
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: narray
   requirement: !ruby/object:Gem::Requirement
@@ -45,84 +45,98 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.6
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
-  type: :development
+        version: '1.1'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: os
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.15'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '3.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.1.0
+        version: '3.4'
 description: "\n    LicenseMatcher is rubygem, which uses Fosslim to match various
   OSS license\n    with correct SPDX-id or EULA label.\n  "
 email:
@@ -142,6 +156,7 @@ files:
 - Rakefile
 - lib/license_matcher.rb
 - lib/license_matcher/native.bundle
+- lib/license_matcher/native.so
 - lib/license_matcher/preprocess.rb
 - lib/license_matcher/rule_matcher.rb
 - lib/license_matcher/tf_ruby_matcher.rb