extractpatterns 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 24ecd7395c9c79e1f035a2420c1e8d6053816d65
+  data.tar.gz: 9d1f1b45a380ff2de4b5cf2b5e7150f943df29a8
+SHA512:
+  metadata.gz: 03b98db5070f0ada7452d5738d5c36eb8bdbb54a51b16e23bae28909fa017417ba62769fab87edee0be432f872b8f0fc5d106b105ced3f80bd6ffb271086f140
+  data.tar.gz: a1e84c75ba367661a8ca80625de90d4b4ee5875a9c47e75cd5072ddfd2b79e6b2302fcebca217dd4bb845e545805a179fa0cf9128c99da4d769f643a552d28f2

data/lib/extractpatterns.rb

@@ -0,0 +1,134 @@
+require 'json'
+require 'pry'
+require 'termextractor'
+
+class ExtractPatterns
+  def initialize(input, fields, match_name)
+    @input = JSON.parse(input)
+    @fields = fields
+    @match_name = match_name
+    @output = Array.new
+  end
+
+  # Split to find matches
+  def comma_list_matches(value)
+    if value
+      # Split on commas
+      list_items = value.split(",")
+
+      # Only get items under certain num of words
+      list_items.reject!{ |item| item.split(" ").length > 2 }
+
+      # Clean whitespace and ands
+      return list_items.map { |match| match.gsub(" and", "").gsub("and ", "").gsub(".", "").strip.lstrip }
+    end
+    return []
+  end
+
+  # Get words in ALLCAPS past certain length
+  def get_allcaps(value, length)
+    if length && value
+      # Get all matches
+      matches = value.scan(/\b(?:[A-Z]|\s){#{length},}\b/)
+      
+      # Remove matches that are too long
+      matches.reject!{|match| match.length > 100}
+      
+      # Remove whitespace
+      return matches.map{ |match| match.strip.lstrip }
+    end
+    return []
+  end
+
+  # Extract set terms
+  def find_known_terms(item, field, extract_list)
+    d = TermExtractor.new(JSON.pretty_generate([item]), [field], "extracted_codewords")
+    d.extractSetTerms(File.read(extract_list), ["codeword"], "case_sensitive")
+    return JSON.parse(d.getAllOutput).first["extracted_codewords"]
+  end
+
+  # Normalize and match synonyms and deduplicate
+  def normalize_results(extracted_raw, synonym_list)
+    synonyms = JSON.parse(File.read(synonym_list))
+    outarr = extracted_raw.dup
+
+    # Go through all extracted
+    extracted_raw.each do |extracted|
+      # Go through each item in synonym list
+      synonyms.each do |key, value|
+        value["codeword"].each do |word|
+          # Match found!
+          if word.downcase == extracted.downcase
+            outarr.delete(extracted)
+            outarr.push(key)
+          end
+        end
+      end
+    end
+
+    # Return deduplicated
+    return outarr.uniq
+  end
+
+  # Go through all items in JSON and fields to search
+  def search_fields(allcaps_length, extract_list, merge_field)
+    # Extract from each item
+    @input.each do |item|
+      item[@match_name] = Array.new
+      
+      @fields.each do |field|
+        # Extract list results, allcaps, and known codewords from each field
+        list_results = comma_list_matches(item[field])
+        allcaps_results = get_allcaps(item[field], allcaps_length)
+        merge_results = item[merge_field] ? item[merge_field] : []
+        known_terms_results = find_known_terms(item, field, extract_list)
+                             
+        # Merge results and post-process
+        item[@match_name] = item[@match_name] | normalize_results((allcaps_results | list_results | merge_results | known_terms_results),
+                                                                  extract_list)
+      end
+
+      # Push updated item out
+      @output.push(item)
+    end
+
+    return @output
+  end
+
+  # Return a ranked hash of the results
+  def ranked_hash_output(results)
+    # Make array of all results
+    allmatches = Array.new
+    results.each do |i|
+      i["tools_mentioned"].each do |match|
+        allmatches.push(match)
+      end
+    end
+
+    # Make ranked hash
+    rankedhash = Hash.new
+    allmatches.each do |match|
+      if rankedhash[match]
+        rankedhash[match] += 1
+      else
+        rankedhash[match] = 1
+      end
+    end
+    return rankedhash.sort_by{|k, v| v}
+  end
+end
+
+#dir = "/home/shidash/Data/unknown_test"
+#overalloutput = Array.new
+#Dir.foreach(dir) do |file|
+#  next if file == '.' or file == '..'
+#  if !File.directory?(dir+"/"+file) && file.include?(".json") && !file.include?(".json.gpg")
+#    e = ExtractPatterns.new(File.read(dir+"/"+file), ["additional_info", "job_description", "skills", "summary"], "tools_mentioned")
+#    results = e.search_fields(6, "extract_list.json", nil)
+#    File.write(file.gsub(".json", "_extracted.json"), JSON.pretty_generate(results))
+#    overalloutput.concat(results)
+#  end
+#end
+
+#e = ExtractPatterns.new(File.read("MECWEDB.json"), ["description", "summary"], "tools_mentioned")
+#puts e.ranked_hash_output(overalloutput)

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: extractpatterns
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-25 00:00:00.000000000 Z
+dependencies: []
+description: Extracts entities and terms from any JSON.
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/extractpatterns.rb
+homepage: https://github.com/transparencytoolkit/ExtractPatterns
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Extracts entities and terms
+test_files: []
+has_rdoc: