inci_score 1.1.0

data/config.ru

@@ -0,0 +1,3 @@
+require 'inci_score/api/app'
+
+run InciScore::API::App

data/ext/levenshtein.c

@@ -0,0 +1,43 @@
+static int call(const char* word1, int len1, const char* word2, int len2) {
+        int matrix[len1 + 1][len2 + 1];
+        int i;
+        for (i = 0; i <= len1; i++) {
+                matrix[i][0] = i;
+        }
+        for (i = 0; i <= len2; i++) {
+                matrix[0][i] = i;
+        }
+        for (i = 1; i <= len1; i++) {
+                int j;
+                char c1;
+
+                c1 = word1[i-1];
+                for (j = 1; j <= len2; j++) {
+                        char c2;
+
+                        c2 = word2[j-1];
+                        if (c1 == c2) {
+                                matrix[i][j] = matrix[i-1][j-1];
+                        }
+                        else {
+                                int delete;
+                                int insert;
+                                int substitute;
+                                int minimum;
+
+                                delete = matrix[i-1][j] + 1;
+                                insert = matrix[i][j-1] + 1;
+                                substitute = matrix[i-1][j-1] + 1;
+                                minimum = delete;
+                                if (insert < minimum) {
+                                        minimum = insert;
+                                }
+                                if (substitute < minimum) {
+                                        minimum = substitute;
+                                }
+                                matrix[i][j] = minimum;
+                        }
+                }
+        }
+        return matrix[len1][len2];
+}

data/inci_score.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'inci_score/version'
+
+Gem::Specification.new do |s|
+  s.name = "inci_score"
+  s.version = InciScore::VERSION
+  s.authors = ["costajob"]
+  s.email = ["costajob@gmail.com"]
+  s.summary = %q{A library that computes the hazard of cosmetic products components, based on the Biodizionario data.}
+  s.homepage = "https://github.com/costajob/inci_score.git"
+  s.license = "MIT"
+  s.required_ruby_version = ">= 2.0.0"
+
+  s.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|test|s|features)/}) }
+  s.bindir = "exe"
+  s.executables = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency "nokogiri", "~> 1.6"
+  s.add_runtime_dependency "puma", "~> 3.0"
+  s.add_runtime_dependency "RubyInline", "~> 3.0"
+
+  s.add_development_dependency "bundler", "~> 1.11"
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "minitest", "~> 5.0"
+  s.add_development_dependency "rack-test", "~> 0.6"
+end

data/lib/inci_score/api/app.rb

@@ -0,0 +1,21 @@
+require 'rack'
+require 'inci_score'
+
+module InciScore
+  module API
+    module App
+      extend self
+
+      def catalog
+        @catalog ||= Catalog.fetch
+      end
+
+      def call(env)
+        req = Rack::Request.new(env)
+        src = req.params["src"]
+        json = src ? Computer.new(src, catalog).call.to_json : %q({"error": "no valid source"})
+        ['200', {'Content-Type' => 'application/json'}, [json]]
+      end
+    end
+  end
+end

data/lib/inci_score/catalog.rb

@@ -0,0 +1,13 @@
+require 'yaml'
+
+module InciScore
+  module Catalog
+    extend self
+
+    YAML_PATH = File::expand_path('../../../config/catalog.yml', __FILE__) 
+
+    def fetch(src = File.read(YAML_PATH))
+      YAML::load(src)
+    end
+  end
+end

data/lib/inci_score/computer.rb

@@ -0,0 +1,45 @@
+require 'inci_score/normalizer'
+require 'inci_score/recognizer'
+require 'inci_score/scorer'
+require 'inci_score/response'
+
+module InciScore
+  class Computer
+    TOLERANCE = 30.0
+
+    def initialize(src, catalog)
+      @src = src
+      @catalog = catalog
+      @unrecognized = []
+    end
+
+    def call
+      @response ||= Response.new(components: components.map(&:first),
+                                 unrecognized: @unrecognized,
+                                 score: score,
+                                 valid: valid?)
+    end
+
+    private
+
+    def score
+      Scorer.new(components.map(&:last)).call
+    end
+
+    def ingredients
+      @ingredients ||= Normalizer.new(src: @src).call
+    end
+
+    def components
+      @components ||= ingredients.map do |ingredient|
+        Recognizer.new(ingredient, @catalog).call.tap do |component|
+          @unrecognized << ingredient unless component
+        end
+      end.compact
+    end
+
+    def valid?
+      @unrecognized.size / (ingredients.size / 100.0) <= TOLERANCE
+    end
+  end
+end

data/lib/inci_score/levenshtein.rb

@@ -0,0 +1,55 @@
+require 'inline'
+
+module InciScore
+  class LevenshteinC
+    C_PROGRAM = File::expand_path('../../../ext/levenshtein.c', __FILE__)
+
+    inline(:C) do |builder|
+      builder.c File::read(C_PROGRAM) 
+    end
+  end
+
+  class Levenshtein
+    def initialize(s, t)
+      @s = s.downcase.unpack("U*")
+      @t = t.downcase.unpack("U*")
+    end
+
+    def call
+      n, m = @s.length, @t.length
+
+      return 0 if @s == @t
+      return m if n.zero?
+      return n if m.zero?
+
+      d = Array.new(m+1) { |i| i }
+      x = nil
+
+      n.times do |i|
+        e = i + 1
+        m.times do |j|
+          c = @s[i] == @t[j] ? 0 : 1
+          ins = d[j + 1] + 1
+          del = e + 1
+          sub = d[j] + c
+          x = ins < del ? ins : del
+          x = sub if sub < x
+          d[j] = e
+          e = x
+        end
+        d[m] = x
+      end
+      x
+    end
+  end
+end
+
+String::class_eval do
+  def distance_utf8(t)
+    InciScore::Levenshtein.new(self, t).call
+  end
+
+  def distance(t)
+    InciScore::LevenshteinC.new.call(self.downcase, self.size, t.downcase, t.size)
+  end
+end

data/lib/inci_score/normalizer.rb

@@ -0,0 +1,21 @@
+require 'inci_score/normalizer_rules'
+
+module InciScore
+  class Normalizer
+    DEFAULT_RULES = Rules.constants - [:Base]
+
+    attr_reader :src
+
+    def initialize(options = {})
+      @src = options[:src] || fail(ArgumentError, 'missing src')
+      @rules = options.fetch(:rules) { DEFAULT_RULES }
+    end
+
+    def call
+      @rules.reduce(@src) do |src, name|
+        rule = Rules.const_get(name).new(src)
+        src = rule.call
+      end
+    end
+  end
+end

data/lib/inci_score/normalizer_rules.rb

@@ -0,0 +1,76 @@
+module InciScore
+  class Normalizer
+    module Rules
+      class Base
+        SEPARATOR = ','
+
+        def initialize(src)
+          @src = src
+        end
+
+        def call
+          fail NotImplementedError
+        end
+      end
+
+      class Replacer < Base
+        REPLACEMENTS = [
+          [/\n+|\t+/, ' '],
+          ['‘', "'"],
+          ['—', '-'],
+          ['(', 'C'],
+          ['_', ' '],
+          ['~', '-'],
+          ['|', 'l'],
+          [' I ', '/']
+        ]
+
+        def call
+          REPLACEMENTS.reduce(@src) do |src, replacement|
+            invalid, valid = *replacement
+            src.index(invalid) ? src.gsub(invalid, valid) : src
+          end
+        end
+      end
+
+      class Downcaser < Base
+        def call 
+          @src.downcase
+        end
+      end 
+
+      class Beheader < Base
+        TITLE_SEP = ':'
+        MAX_INDEX = 50
+
+        def call
+          sep_index = @src.index(TITLE_SEP)
+          return @src if !sep_index || sep_index > MAX_INDEX
+          @src[sep_index+1, @src.size]
+        end
+      end
+
+      class Separator < Base
+        SEPARATORS = ["; ", ". ", " ' ", " - ", " : "]
+
+        def call
+          SEPARATORS.reduce(@src) do |src, separator|
+            src = src.gsub(separator, SEPARATOR)
+          end
+        end
+      end 
+
+      class Tokenizer < Base
+        INVALID_CHARS = /[^\w\s-]/
+
+        def call
+          @src.split(SEPARATOR).map do |token|
+            token = token.sub(/\/.*/, '')
+            token = token.gsub(INVALID_CHARS, '')
+            token = token.strip
+          end.reject(&:empty?)
+        end
+      end
+    end
+  end
+end

data/lib/inci_score/parser.rb

@@ -0,0 +1,43 @@
+require 'nokogiri'
+
+module InciScore
+  class Parser
+    BIODIZIO_URI = 'http://www.biodizionario.it/biodizio.php'
+    SEMAPHORES = %w[vv v g r rr]
+    CSS_QUERY = 'table[width="751"] > tr > td img'
+
+    def initialize(src = nil)
+      @src = src || Thread.new { open(BIODIZIO_URI) }
+    end
+
+    def call
+      @components ||= Nokogiri::HTML(doc).css(CSS_QUERY).inject({}) do |acc, img|
+        hazard = semaphore(img.attr('src'))
+        name = img.next_sibling.next_sibling
+        desc = name.next_sibling.next_sibling
+        name, desc = desc, name if swap?(desc.text)
+        acc[normalize(name)] = SEMAPHORES.index(hazard)
+        acc
+      end
+    end
+
+    private
+
+    def doc
+      @src.respond_to?(:value) ? @src.value : @src
+    end
+
+    def semaphore(src)
+      src.match(/(#{SEMAPHORES.join('|')}).gif$/)[1]
+    end
+
+    def normalize(node)
+      node.text.strip.downcase
+    end
+
+    def swap?(desc)
+      return false if desc.empty?
+      desc == desc.upcase
+    end
+  end
+end

data/lib/inci_score/recognizer.rb

@@ -0,0 +1,27 @@
+require 'inci_score/recognizer_rules'
+
+module InciScore
+  class Recognizer
+    DEFAULT_RULES = Rules.constants - [:Base]
+
+    def initialize(src, catalog, rules = DEFAULT_RULES)
+      @src = src
+      @catalog = catalog
+      @rules = rules
+    end
+
+    def call
+      @component = apply_rules
+      return [@component, @catalog[@component]] if @component
+    end 
+
+    private
+
+    def apply_rules
+      @rules.reduce(nil) do |component, name|
+        rule = Rules.const_get(name).new(@src, @catalog)
+        component || rule.call
+      end
+    end
+  end
+end

data/lib/inci_score/recognizer_rules.rb

@@ -0,0 +1,75 @@
+require 'inci_score/levenshtein'
+
+module InciScore
+  class Recognizer
+    module Rules
+      class Base
+        TOLERANCE = 3
+
+        def initialize(src, catalog)
+          @src = src
+          @catalog = catalog
+        end
+
+        def call
+          fail NotmplementedError
+        end
+      end
+
+      class Key < Base
+        def call
+          @src if @catalog.has_key?(@src)
+        end
+      end
+
+      class Levenshtein < Base
+        ALTERNATE_SEP  = '/'
+
+        def call
+          size = @src.size
+          initial = @src[0]
+          component, distance = @catalog.reduce([nil, size]) do |min, (comp, _)|
+            next min unless comp.start_with?(initial)
+            match = (n = comp.index(ALTERNATE_SEP)) ? comp[0, n] : comp
+            next min if match.size > (size + TOLERANCE)
+            dist = @src.distance(match)
+            min = [comp, dist] if dist < min[1]
+            min
+          end
+          component unless distance > TOLERANCE || distance >= (size-1)
+        end
+      end
+
+      class Digits < Base
+        MIN_MEANINGFUL = 7
+
+        def call
+          return if @src.size < TOLERANCE
+          digits = @src[0, MIN_MEANINGFUL]
+          @catalog.detect do |component, _| 
+            component.match(/^#{Regexp::escape(digits)}/)
+          end.to_a.first
+        end
+      end
+
+      class Tokens < Base
+        UNMATCHABLE = %w[extract oil sodium acid sulfate]
+      
+        def call
+          tokens.each do |token|
+            @catalog.each do |component, _| 
+              return component if component.match(/\b#{Regexp.escape(token)}\b/)
+            end
+          end
+          nil
+        end
+
+        private 
+        
+        def tokens
+          (@src.split(' ') - UNMATCHABLE).reject { |t| t.size < TOLERANCE }.sort_by!(&:size).reverse!
+        end
+      end
+    end
+  end
+end

data/lib/inci_score/response.rb

@@ -0,0 +1,31 @@
+require 'json'
+
+module InciScore
+  class Response
+    attr_reader :components, :score, :unrecognized, :valid
+
+    def initialize(options = {})
+      @components = options.fetch(:components) { [] }
+      @score = options.fetch(:score) { 0.0 }
+      @unrecognized = options.fetch(:unrecognized) { [] }
+      @valid = options.fetch(:valid) { false }
+    end
+
+    def to_json
+      { components: @components, unrecognized: @unrecognized, score: @score, valid: @valid }.to_json
+    end
+
+    def to_s
+      %Q{
+TOTAL SCORE:
+      \t#{@score}
+VALID STATE:
+      \t#{@valid}
+COMPONENTS:
+      #{@components.map { |c| "\t#{c}" }.join("\n")}
+UNRECOGNIZED:
+      #{@unrecognized.map { |c| "\t#{c}" }.join("\n")}
+      }
+    end
+  end
+end

data/lib/inci_score/score.rb

@@ -0,0 +1,19 @@
+module InciScore
+  class Score
+    attr_reader :value
+
+    def initialize(hazard, weight)
+      @hazard = hazard
+      @weight = weight
+      @value = compute
+    end
+
+    private
+
+    def compute
+      (@hazard - @weight).tap do |s|
+        return 0.0 if s < 0
+      end
+    end
+  end
+end

data/lib/inci_score/scorer.rb

@@ -0,0 +1,45 @@
+require 'inci_score/score'
+
+module InciScore
+  class Scorer
+    HAZARD_PERCENT = 25
+    WEIGHT_FACTOR = 5
+
+    def initialize(hazards)
+      @hazards = Array(hazards)
+      @size = @hazards.size
+    end
+
+    def call
+      return 0 if @hazards.empty?
+      100 - avg * HAZARD_PERCENT
+    end
+
+    private
+
+    def avg
+      avg_weighted / @size.to_f
+    end
+
+    def avg_weighted
+      return @hazards.reduce(&:+) if same_hazard?
+      weighted.reduce(0.0) do |acc,score| 
+        acc += score.value
+      end
+    end
+
+    def same_hazard?
+      @hazards.uniq.size == 1
+    end
+
+    def weighted
+      @hazards.each_with_index.map do |h,i|
+        Score.new(h, weight(i))
+      end
+    end
+
+    def weight(index)
+      Math.log(index+1, @size * WEIGHT_FACTOR)
+    end
+  end
+end

data/lib/inci_score/version.rb

@@ -0,0 +1,3 @@
+module InciScore
+  VERSION = "1.1.0"
+end

data/lib/inci_score.rb

@@ -0,0 +1,4 @@
+require 'inci_score/version'
+require 'inci_score/parser'
+require 'inci_score/catalog'
+require 'inci_score/computer'

data/log/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore