termextractor 0.0.16

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b96a8a2568c9b0f2bb0b17b07a6d0131b39ed0e5
+  data.tar.gz: 2f1612dfd0564eda138957fd50d0a6b058483b15
+SHA512:
+  metadata.gz: 2b70ffec27aa30dc168f674175693bd1dd31f0ed044af7c10ea2a39591285a373768e8b8e85828e8e07f5ec38a2316763885178368f86b1acf1f3fa134610f9e
+  data.tar.gz: 0c0bb009fa834204d765965aca672b2f07cbef92f8821d3edae4aa1b58df73c7dd0f2bffb0fabe058794b58cf0afbb25175df46494a5880e88150298a5f9daf0

data/lib/extract_set_terms.rb

@@ -0,0 +1,92 @@
+class ExtractSetTerms
+  def initialize(item, extract_field, to_extract, extract_term_fields, case_sensitive, save_field)
+    @item = item
+    @extract_field = extract_field
+
+    @to_extract = JSON.parse(to_extract)
+    @extract_term_fields = extract_term_fields
+    @case_sensitive = case_sensitive
+
+    @extract_dict = Hash.new
+    @save_field = save_field
+    @item_out = item
+  end
+
+  # Gets a list of terms to extract
+  def processHashInput
+    # Go through each item then each field
+    @to_extract.each do |ex_key, ex_value|
+      ex_value.each do |ex_field, ex_term|
+        
+        # Check if it is the right field
+        if ex_field == @extract_term_fields || @extract_term_fields.include?(ex_field)
+          # Make dictionary of terms to extract and overall mapping
+          ex_term.is_a?(Array) ? processArrayInput(ex_term, ex_key) : @extract_dict[term] = ex_key
+        end
+
+      end
+    end
+  end
+
+  # Add all items in array to dictionary of terms to extract
+  def processArrayInput(extract_arr, map_val)
+    extract_arr.each do |term| 
+      map_val = term if map_val == nil
+      @extract_dict[term] = map_val
+    end
+  end
+
+  # Check if the term appears in the text
+  def matchTerm?(term, text, case_sensitive)
+    # Downcase term and text if not case sensitive
+    if case_sensitive == false
+      term = term.downcase
+      text = text.downcase
+    end
+    
+    # Return if it maches
+    if text.to_s.match(/\b(#{term})\b/)
+      return true
+    end
+  end
+
+  # Check if item is case sensitive
+  def isCaseSensitive?(term)
+    if @case_sensitive == "case-sensitive"
+      return true
+    elsif @case_sensitive == "case-insensitive"
+      return false
+    else
+      # Handle item by item variations
+      is_case_sensitive = @to_extract[term[1]][@case_sensitive]
+      if is_case_sensitive == "Yes"
+        return true 
+      else return false
+      end
+    end
+  end
+
+  # Process input list and go through all terms and fields
+  def extractTerms
+    # Process input list
+    @to_extract.is_a?(Hash) ? processHashInput : processArrayInput(@to_extract, nil)
+    @item_out[@save_field] = Array.new
+
+    # Go through each term and field to check for matches
+    @extract_dict.each do |term|
+      item_case_sensitivity = isCaseSensitive?(term)
+      @extract_field.each do |field|
+
+        # Add to list of terms if it matches
+        if matchTerm?(term[0], @item[field], item_case_sensitivity)
+          @item_out[@save_field].push(term[1])
+        end
+
+      end
+    end
+
+    # Deduplicate and return
+    @item_out[@save_field].uniq!
+    return @item_out
+  end
+end

data/lib/term_extractor.rb

@@ -0,0 +1,46 @@
+require 'json'
+load 'extract_set_terms.rb'
+
+class TermExtractor
+  def initialize(input, extract_from, save_field)
+    @input = JSON.parse(input)
+    @extract_from = extract_from
+    @save_field = save_field
+
+    @output = Array.new
+  end
+
+  # Extracts set terms
+  def extractSetTerms(to_extract, extract_term_fields, case_sensitive)
+    @input.each do |item|
+      extract = ExtractSetTerms.new(item, @extract_from, to_extract, extract_term_fields, case_sensitive, @save_field)
+      @output.push(extract.extractTerms)
+    end
+  end
+
+  # Gets all results in output
+  def getAllOutput
+    JSON.pretty_generate(@output)
+  end
+
+  # Gets only the results for which terms were found/extracted
+  def getOnlyMatching
+    matches = @output.select { |item| !item[@save_field].empty? }
+    JSON.pretty_generate(matches)
+  end
+
+  # Gets a list of the extracted terms by how often they occur
+  def getTermList
+    counthash = Hash.new{0}
+
+    # Increments for each occurrence of term
+    @output.each do |item|
+      item[@save_field].each do |term|
+        counthash[term] += 1
+      end
+    end
+
+    # Return hash sorted by value
+    return Hash[counthash.sort_by { |k, v| v}]
+  end
+end

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: termextractor
+version: !ruby/object:Gem::Version
+  version: 0.0.16
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-20 00:00:00.000000000 Z
+dependencies: []
+description: Extracts entities and terms from any JSON.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/extract_set_terms.rb
+- lib/term_extractor.rb
+homepage: https://github.com/Shidash/EntityExtractor
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Extracts entities and terms
+test_files: []