harmonizer_redis 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1b7c2cbd0cf8dbb603f66c6b7dc37fe6cb780cbf
+  data.tar.gz: e803d14a56ca973ac52e02e6804c6ee5965cb6c0
+SHA512:
+  metadata.gz: 551d4c226fa3cf6f8badd822f7442b6d7c5673e09fb4687f169afe8e57da82f686c6bd0084cf68f714feadecdf53274c89aee08d0a99c55d46a84ad4f24d1b2b
+  data.tar.gz: 9cf6d8c7ddce775cda2dff601421c961fc2326e0c9233805d3d21a99f782653feb70b19b30f6b374ddef76ea8a51691acb77b3478261946d0152ac286e5cc39e

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+Makefile
+*.bundle
+.DS_Store
+.idea/
+*.gem

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.0
+before_install: gem install bundler -v 1.10.4

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in harmonizer_redis.gemspec
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Tian Wang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# HarmonizerRedis
+
+HarmonizerRedis is a Ruby gem that aids the process of relabeling/grouping free text phrases to
+resolve the many ways people spell or describe something. It uses fuzzy string matching along with inverse
+term frequencies to score and rank similarities between phrases. The gem uses Redis for performance.
+
+## Usage
+
+### Configuration
+
+The Redis must be configured first. Refer to the [Redis] (https://github.com/redis/redis-rb) for more information.
+`Redis.current` should be set to the Redis connection.
+
+```ruby
+Redis.current = Redis.new
+```
+
+### Adding an entry
+
+`HarmonizerRedis::Linkage` represents the connection between your data structures and the gem. Linkages contain
+string content, an `id` (which will be a uniquely generated uuid), and a `category_id` which identifies the collection this entry belongs to.
+
+```ruby
+my_category_id = 100
+linkage = HarmonizerRedis::Linkage.new(content: 'harmonizer redis',
+                                       category_id: my_category_id)
+linkage.save
+my_linkage_id = linkage.id # "520c488b-e9f8-4a6f-aaea-0d5e37b97644"
+```
+
+### Retrieving an entry
+
+```ruby
+my_linkage = HarmonizerRedis::Linkage.find(my_linkage_id)
+```
+
+### Calculating and Retrieving Similarities
+
+Calculate similarities for all the linkages in a category in a batch. New calculations will need to
+be performed if new linkages are added.
+
+```ruby
+HarmonizerRedis.calculate_similarities(my_category_id)
+```
+
+To get an Array of similar phrases. The default is to return the top 20 phrases. If new linkages have
+been added or if the similarities have not yet been computed for this linkage, it will be computed
+automatically with this call.
+
+```ruby
+my_linkage.get_similarities
+```
+
+### Merging into groups, labeling groups, and getting recommended labels
+
+Each entry in this array is an array in the following format `[text_label, group_label, similarity_score, phrase_id]`
+
+After deciding which phrase the linkage should be combined with - use the accompanying phrase_id data to merge the phrases into a group
+
+```ruby
+my_linkage.merge_with_phrase(phrase_id)
+```
+
+To label everything in the same group:
+
+```ruby
+my_linkage.set_corrected_label('HarmonizerRedis')
+```
+
+To suggest labels for this group (this works better the more HarmonizerRedis is used)
+
+```ruby
+my_linkage.recommend_labels
+```
+
+Lastly to get the final corrected label of a linkage:
+
+```ruby
+my_linkage.corrected
+```
+
+## Contributing
+
+Feel free to fork this repo and change it as you wish. We prefer pull requests on github, but you can send us emails. All attributions need to be tested as well.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,19 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rake/extensiontask"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec
+task :test    => :spec
+Rake::Task[:test].prerequisites << :compile
+
+task :console do
+  exec "irb -r harmonizer_redis -I ./lib"
+end
+
+Rake::ExtensionTask.new('white_similarity') do |extension|
+  extension.lib_dir = 'lib/harmonizer_redis'
+end
+
+task :build => [:clean, :compile]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "harmonizer_redis"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/ext/white_similarity/extconf.rb

@@ -0,0 +1,5 @@
+require 'mkmf'
+
+$CFLAGS = '--std=c99 -O'
+
+create_makefile('harmonizer_redis/white_similarity')

data/ext/white_similarity/white_similarity.c

@@ -0,0 +1,159 @@
+#include "ruby.h"
+#include <stdio.h>
+#include <string.h>
+#include <stdbool.h>
+#include "white_similarity.h"
+
+void Init_white_similarity()
+{
+    WhiteSimilarity = rb_define_module("WhiteSimilarity");
+    rb_define_method(WhiteSimilarity, "score", method_score, 2);
+    rb_define_method(WhiteSimilarity, "soft_cos_similarity", method_soft_cos_similarity, 2);
+}
+
+inline bool makes_bad_pair(char *str, int index)
+{
+    return (str[index] == ' ' || str[index+1] == ' ');
+}
+
+inline bool is_pair_equal(char *x_pair, char *y_pair)
+{
+    return (x_pair[0] == y_pair[0] && x_pair[1] == y_pair[1]);
+}
+
+struct PairHolder {
+    int pair_count;
+    char **pairs;
+};
+
+Pair generate_pairs(char *str)
+{
+    Pair str_pairs;
+    int max_pair_number = strlen(str) - 1;
+    str_pairs.pairs = malloc(max_pair_number * sizeof(char *));
+
+    int pair_count = 0;
+    for (int i = 0; i < max_pair_number; i++) {
+        if (!makes_bad_pair(str, i))
+        {
+            str_pairs.pairs[pair_count] = str + i;
+            pair_count++;
+        }
+    }
+    str_pairs.pair_count = pair_count;
+    return str_pairs;
+}
+
+double white_similarity(char *x_str, char *y_str)
+{
+    Pair x_pairs = generate_pairs(x_str);
+    Pair y_pairs = generate_pairs(y_str);
+    int intersect = 0;
+    int sum = x_pairs.pair_count + y_pairs.pair_count;
+
+    for (int i = 0; i < x_pairs.pair_count; i++) {
+        for (int j = 0; j < y_pairs.pair_count; j++) {
+            if (x_pairs.pairs[i] != NULL && y_pairs.pairs[j] != NULL &&
+                is_pair_equal(x_pairs.pairs[i], y_pairs.pairs[j])) {
+                intersect++;
+                y_pairs.pairs[j] = NULL;
+                break;
+            }
+        }
+    }
+    free(x_pairs.pairs);
+    free(y_pairs.pairs);
+
+    return 2.0 * (double)intersect / (double)sum;
+}
+
+VALUE method_score(VALUE self, VALUE x_string, VALUE y_string)
+{
+    char *x_str = StringValueCStr(x_string);
+    char *y_str = StringValueCStr(y_string);
+    if (strlen(x_str) == 1 || strlen(y_str) == 1) {
+        return rb_float_new(0.0);
+    }
+    double similarity = white_similarity(x_str, y_str);
+    return rb_float_new(similarity);
+}
+
+/* code needed for soft cos similarity calculations */
+
+struct MatrixHolder {
+    int len;
+    char *raw_string;
+    char **words;
+    double *values;
+};
+
+Matrix generate_matrix(char *matrix_cstr)
+{
+    Matrix new_matrix;
+
+    new_matrix.raw_string = malloc(strlen(matrix_cstr + 1) + 1);
+    strcpy(new_matrix.raw_string, matrix_cstr + 1);
+
+    //printf("%s", new_matrix.raw_string);
+
+    new_matrix.len = (int)matrix_cstr[0];
+    new_matrix.words = malloc(new_matrix.len * sizeof(char *));
+    new_matrix.values = malloc(new_matrix.len * sizeof(double));
+
+    char *head = new_matrix.raw_string;
+    for (int i = 0; i < new_matrix.len; i++) {
+        new_matrix.words[i] = head;
+        while (*head != ',') {
+            head++;
+        }
+        *head = '\0';
+        head++;
+    }
+
+    for (int i = 0; i < new_matrix.len; i++) {
+        char *curr_value = head;
+        while(*head != ',' && *head != '\0') {
+            head++;
+        }
+        if (*head == ',') {
+            *head = '\0';
+            head++;
+        }
+        sscanf(curr_value, "%lf", new_matrix.values + i);
+    }
+
+    return new_matrix;
+}
+
+void free_matrix(Matrix to_free)
+{
+    free(to_free.raw_string);
+    free(to_free.words);
+    free(to_free.values);
+}
+
+double soft_cos_similarity(char *x_matrix_str, char *y_matrix_str)
+{
+    Matrix x_matrix = generate_matrix(x_matrix_str);
+    Matrix y_matrix = generate_matrix(y_matrix_str);
+    double similarity = 0.0;
+
+    for (int i = 0; i < x_matrix.len; i++) {
+        for (int j = 0; j < y_matrix.len; j++) {
+            double word_sim = white_similarity(x_matrix.words[i], y_matrix.words[j]);
+            similarity += (word_sim * x_matrix.values[i] * y_matrix.values[j]);
+        }
+    }
+
+    free_matrix(x_matrix);
+    free_matrix(y_matrix);
+
+    return similarity;
+}
+
+VALUE method_soft_cos_similarity(VALUE self, VALUE x_matrix, VALUE y_matrix)
+{
+    char *x_matrix_cstr = StringValueCStr(x_matrix);
+    char *y_matrix_cstr = StringValueCStr(y_matrix);
+    return rb_float_new(soft_cos_similarity(x_matrix_cstr, y_matrix_cstr));
+}

data/ext/white_similarity/white_similarity.h

@@ -0,0 +1,16 @@
+VALUE WhiteSimilarity = Qnil;
+void Init_white_similarity();
+
+typedef struct PairHolder Pair;
+typedef struct MatrixHolder Matrix;
+
+VALUE method_score(VALUE self, VALUE x_string, VALUE y_string);
+double white_similarity(char *x_str, char *y_str);
+Pair generate_pairs(char *str);
+inline bool makes_bad_pair(char *str, int index);
+inline bool is_pair_equal(char *x_pair, char *y_pair);
+
+Matrix generate_matrix(char *matrix_cstr);
+void free_matrix(Matrix to_free);
+double soft_cos_similarity(char *x_matrix_str, char *y_matrix_str);
+VALUE method_soft_cos_similarity(VALUE self, VALUE x_matrix, VALUE y_matrix);

data/harmonizer_redis.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'harmonizer_redis/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "harmonizer_redis"
+  spec.version       = HarmonizerRedis::VERSION
+  spec.authors       = ["Tian Wang"]
+  spec.email         = ["twang95@stanford.edu"]
+
+  spec.summary       = %q{Harmonizes records}
+  spec.description   = %q{Harmonizes records based on fuzzy string/phrase matching. Built on redis for speed}
+  spec.homepage      = "https://github.com/POSpulse/harmonizer_redis"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+  spec.add_dependency "hiredis"
+  spec.add_dependency "activesupport"
+  spec.extensions    = ["ext/white_similarity/extconf.rb"]
+
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/harmonizer_redis.rb

@@ -0,0 +1,59 @@
+require 'harmonizer_redis/version'
+require 'harmonizer_redis/base_object'
+require 'harmonizer_redis/linkage'
+require 'harmonizer_redis/phrase'
+require 'harmonizer_redis/idf_scorer'
+require 'harmonizer_redis/white_similarity'
+require 'harmonizer_redis/category'
+require 'active_support/all'
+require 'redis/connection/hiredis'
+require 'redis'
+
+include WhiteSimilarity
+
+module HarmonizerRedis
+  ### Calculate Similarities. Store them with the category
+  class << self
+    def calculate_similarities(category_id)
+      unless Category.valid?(category_id)
+        raise "Category ID: #{category_id} is invalid"
+      end
+
+      phrase_id_list = Category.get_phrase_list(category_id)
+
+      matrix_list = Category.get_matrices(category_id, phrase_id_list)
+
+      Redis.current.pipelined do
+        (0...phrase_id_list.length).each do |i|
+          (i + 1...phrase_id_list.length).each do |j|
+            id_x = phrase_id_list[i]
+            id_y = phrase_id_list[j]
+            score = Phrase.calc_soft_pair_similarity(matrix_list[i], matrix_list[j])
+            unless score < 0.2
+              add_similarity_entry(id_x, id_y, score, category_id)
+            end
+          end
+        end
+      end
+
+      Category.reset_changed(category_id)
+    end
+
+
+    ### String PreProcessing
+    def normalize_string(string)
+      ActiveSupport::Inflector.transliterate(string.strip.downcase).
+          split(/[^\p{L}0-9]/).delete_if { |x| x.length == 0 }.join(' ')
+    end
+
+    ### Helper
+
+    def add_similarity_entry(id_x, id_y, score, category_id)
+      Redis.current.zadd("HarmonizerRedis::Category:#{category_id}:#{id_x}:sims", score, id_y)
+      Redis.current.zadd("HarmonizerRedis::Category:#{category_id}:#{id_y}:sims", score, id_x)
+    end
+
+    private :add_similarity_entry
+  end
+
+end

data/lib/harmonizer_redis/base_object.rb

@@ -0,0 +1,24 @@
+module HarmonizerRedis
+  class BaseObject
+    attr_accessor :id
+
+    def generate_id
+      Redis.current.incr("#{self.class}").to_i - 1
+    end
+
+    def save
+      #creates a new id only when object is being saved
+      klass = "#{self.class}"
+      new_id = @id || self.generate_id
+      self.instance_variables.each do |variable|
+        var_name = variable.to_s[1..-1]
+        Redis.current.set("#{klass}:#{new_id}:#{var_name}", instance_variable_get(variable))
+      end
+
+      @id = new_id
+
+      #add id to HarmonizerRedis::ClassName:set
+      Redis.current.sadd("#{klass}:set", @id)
+    end
+  end
+end

data/lib/harmonizer_redis/category.rb

@@ -0,0 +1,236 @@
+module HarmonizerRedis
+  class Category < BaseObject
+    attr_reader :id
+    def initialize(id)
+      @id = id
+    end
+
+    def save
+      super()
+    end
+
+    class << self
+      # Add linkage to category group
+      def add_linkage(linkage)
+        category_id = linkage.category_id
+        linkage_id = linkage.id
+        phrase_id = linkage.phrase_id
+
+        unless self.valid?(category_id)
+          new_category = self.new(category_id)
+          new_category.save
+        end
+
+        self.add_to_linkage_set(category_id, linkage_id)
+        self.add_to_phrase_set(category_id, phrase_id)
+
+        set_changed(category_id, 1)
+        set_calculated(category_id, 0)
+        set_phrase_calculated(category_id, phrase_id, 0)
+
+        # Creating/adding to a group
+        add_group(category_id, phrase_id)
+
+        # Adding linkage to the phrase
+        Phrase.add_linkage(phrase_id, linkage_id, category_id)
+      end
+
+      # Gets list of linkages included in category group
+      def get_linkage_list(category_id)
+        Redis.current.smembers("#{self}:#{category_id}:linkage_set")
+      end
+
+      def get_phrase_list(category_id)
+        Redis.current.smembers("#{self}:#{category_id}:phrase_set")
+      end
+
+      def get_matrices(category_id, phrase_id_list)
+        matrices_list = []
+        if !matrices_calculated?(category_id) || changed?(category_id)
+          phrase_id_list.each do |id|
+            content = Phrase.get_content(id)
+            new_matrix = IdfScorer.serialize_matrix(IdfScorer.calc_soft_matrix(content))
+            Redis.current.set("HarmonizerRedis::Phrase:#{id}:matrix", new_matrix)
+            matrices_list << new_matrix
+          end
+          set_calculated(category_id, 1)
+        else
+          phrase_id_list.each do |phrase_id|
+            matrices_list << Phrase.get_matrix(phrase_id)
+          end
+        end
+        matrices_list
+      end
+
+      # Set the category as "unchanged". Should be called after Category similarities
+      # have been calculated
+      def reset_changed(category_id)
+        set_changed(category_id, 0)
+      end
+
+      # Check to see if id is valid
+      def valid?(category_id)
+        Redis.current.sismember("#{self}:set", "#{category_id}")
+      end
+
+      def changed?(category_id)
+        !Redis.current.getbit("#{self}:changed", category_id).zero?
+      end
+
+      def matrices_calculated?(category_id)
+        !Redis.current.getbit("#{self}:calculated", category_id).zero?
+      end
+
+      def is_phrase_calculated?(category_id, phrase_id)
+        !Redis.current.getbit("#{self}:#{category_id}:calculated", phrase_id).zero?
+      end
+
+      # Merge 2 phrases' groups
+      def merge_phrase_groups(category_id, phrase_a_id, phrase_b_id)
+        group_a = get_group_key(category_id, phrase_a_id)
+        group_b = get_group_key(category_id, phrase_b_id)
+        # Error either group does not exist
+        if group_a.nil? || group_b.nil?
+          raise 'Invalid Phrase ID(s) given!'
+        end
+        # Do nothing if both are already in the same group
+        if group_a == group_b
+          return
+        end
+        label_a = get_group_label(category_id, phrase_a_id)
+        label_b = get_group_label(category_id, phrase_b_id)
+
+        # if label_a and label_b both exist
+        unless label_a.nil? ^ label_b.nil?
+          # if label_a and label_b are not the same label
+          if label_a != label_b
+            # delete both labels due to conflict
+            Redis.current.del(label_a, label_b)
+          else # both labels are the same
+            # delete only the label that belongs to the group getting destroyed
+            Redis.current.del(label_b)
+          end
+        end
+
+        # if only label_b exists
+        if label_a.nil? && !label_b.nil?
+          merge_phrase_group_helper(category_id, group_a, group_b)
+          Redis.current.del(group_a)
+        else # if only label_a exists
+          merge_phrase_group_helper(category_id, group_b, group_a)
+          Redis.current.del(group_b)
+        end
+
+      end
+
+      ### Getting popular linkages and generating possible labels
+
+      def get_group_popular_linkages(category_id, phrase_id, number = 5)
+        phrases = get_group(category_id, phrase_id)
+        linkages = []
+        phrases.each do |id|
+          linkages += Phrase.get_popular_linkages(id)
+        end
+        linkages.sort_by! { |entry| -1 * entry[-1] }
+        linkages.first(number)
+      end
+
+      def get_all_group_labels(category_id, phrase_id)
+        phrases_in_group = get_group(category_id, phrase_id)
+        categories = []
+        phrases_in_group.each do |phrase_id|
+          categories += Phrase.get_categories(phrase_id)
+        end
+        labels = Hash.new { |hash, key| hash[key] = 0.0 }
+        categories.each do |category_id|
+          phrases_in_group.each do |phrase_id|
+            group_label = get_group_label(category_id, phrase_id)
+            unless group_label.nil?
+              labels[group_label] += Phrase.get_linkage_count(phrase_id)
+            end
+          end
+        end
+        labels.to_a.sort_by { |x| x[-1] }
+      end
+
+      def set_group_label(category_id, phrase_id, label)
+        Redis.current.set("#{get_group_key(category_id, phrase_id)}:label", label)
+      end
+
+      def get_group_label(category_id, phrase_id)
+        Redis.current.get("#{get_group_key(category_id, phrase_id)}:label")
+      end
+
+      def get_group_count(category_id, phrase_id)
+        Redis.current.scard(get_group_key(category_id, phrase_id))
+      end
+
+      def get_group(category_id, phrase_id)
+        Redis.current.smembers(get_group_key(category_id, phrase_id))
+      end
+
+      def in_same_group?(category_id, phrase_a_id, phrase_b_id)
+        get_group_key(category_id, phrase_a_id) == get_group_key(category_id, phrase_b_id)
+      end
+
+      ### Helpers ####
+      def add_to_linkage_set(category_id, linkage_id)
+        Redis.current.sadd("#{self}:#{category_id}:linkage_set", linkage_id)
+      end
+
+      def add_to_phrase_set(category_id, phrase_id)
+        Redis.current.sadd("#{self}:#{category_id}:phrase_set", phrase_id)
+      end
+
+      def set_changed(category_id, value)
+        Redis.current.setbit("#{self}:changed", category_id, value)
+      end
+
+      def set_calculated(category_id, value)
+        Redis.current.setbit("#{self}:calculated", category_id, value)
+      end
+
+      def set_phrase_calculated(category_id, phrase_id, value)
+        Redis.current.setbit("#{self}:#{category_id}:calculated", phrase_id, value)
+      end
+
+      def add_group(category_id, phrase_id)
+        group_key = get_group_key(category_id, phrase_id)
+        if group_key.nil?
+          group_key = create_group(category_id, phrase_id)
+          Redis.current.sadd(group_key, phrase_id)
+        end
+      end
+
+      def create_group(category_id, phrase_id)
+        new_group_id = Redis.current.incr("#{self}:#{category_id}:group_count") - 1
+        new_group_key = "#{self}:#{category_id}:group:#{new_group_id}"
+        set_phrase_group(category_id, phrase_id, new_group_key)
+        new_group_key
+      end
+
+      def get_group_key(category_id, phrase_id)
+        Redis.current.get("#{self}:#{category_id}:#{phrase_id}:group")
+      end
+
+      def change_phrases_group(category_id, old_group_key, new_group_key)
+        phrase_list = Redis.current.smembers(old_group_key)
+        phrase_list.each do |phrase_id|
+          set_phrase_group(category_id, phrase_id, new_group_key)
+        end
+      end
+
+      def merge_phrase_group_helper(category_id, source_group, dest_group)
+        Redis.current.sunionstore(dest_group, source_group, dest_group)
+        change_phrases_group(category_id, source_group, dest_group)
+        Redis.current.del(source_group)
+      end
+
+      def set_phrase_group(category_id, phrase_id, group_key)
+        Redis.current.set("#{self}:#{category_id}:#{phrase_id}:group", group_key)
+      end
+
+    end
+    private_class_method :set_changed, :create_group, :get_group_key, :set_phrase_group, :change_phrases_group
+  end
+end

data/lib/harmonizer_redis/idf_scorer.rb

@@ -0,0 +1,113 @@
+module HarmonizerRedis
+  module IdfScorer
+    # class self
+    class << self
+      def add_document(phrase_id)
+        self.incr_doc_count
+        text = HarmonizerRedis::Phrase.get_content(phrase_id)
+        word_set = Set.new
+        text.split.each do |word|
+          unless word_set.include? word
+            word_set.add(word)
+            Redis.current.incr(word_doc_freq_key(word))
+          end
+          Redis.current.incr(word_count_key(word))
+        end
+      end
+
+      def get_score(word)
+        doc_freq = self.get_doc_freq(word) + 0.1
+        doc_count = self.doc_count + 0.1
+        Math.log(0.1+(doc_count / doc_freq))
+      end
+
+      # first char is length of phrase (capped at 255 words). Each word(key) is separated
+      # by a comma and the keys and values are separated by a vertical bar
+      def serialize_matrix(matrix)
+        serialized = "#{matrix.length.chr}#{matrix.keys.join(',')},#{matrix.values.join(',')}"
+        serialized
+      end
+
+      # Used for soft cosine similarity
+      def calc_soft_matrix(phrase_content)
+        matrix = Hash.new(0.0)
+        phrase_content.split.each do |word|
+          matrix[word] += 1.0
+        end
+
+        matrix.each do |word, count|
+          updated = (1.0 + Math::log10(count)) * self.get_score(word)
+          matrix[word] = updated
+        end
+
+        #calculate normalization factor
+        norm_factor_sqrd = 0.0
+        matrix.each do |word_a, value_a|
+          matrix.each do |word_b, value_b|
+            similarity = WhiteSimilarity.score(word_a, word_b)
+            norm_factor_sqrd += (similarity * value_a * value_b)
+          end
+        end
+
+        #normalize
+        factor = Math::sqrt(norm_factor_sqrd)
+        matrix.each do |word, value|
+          matrix[word] = value / factor
+        end
+        matrix
+      end
+
+      def cos_similarity(matrix_a, matrix_b)
+        similarity = 0.0
+        matrix_a.each do |word, value|
+          similarity += (value * matrix_b[word])
+        end
+        similarity
+      end
+
+      def soft_cos_similarity(matrix_a, matrix_b)
+        similarity = 0.0
+        matrix_a.each do |word_a, value_a|
+          matrix_b.each do |word_b, value_b|
+            if word_a != word_b
+              white_similarity = WhiteSimilarity.score(word_a, word_b)
+            else
+              white_similarity = 1.0
+            end
+            similarity += (white_similarity * value_a * value_b)
+          end
+        end
+        similarity
+      end
+
+      def get_doc_freq(word)
+        Redis.current.get(word_doc_freq_key(word)).to_f
+      end
+
+      def get_count(word)
+        Redis.current.get(word_count_key(word)).to_f
+      end
+
+      def decr_doc_freq(word)
+        Redis.current.decr(word_doc_freq_key(word))
+        Redis.current.decr("#{self}:doc_count")
+      end
+
+      def doc_count
+        Redis.current.get("#{self}:doc_count").to_f
+      end
+
+      def incr_doc_count
+        Redis.current.incr("#{self}:doc_count")
+      end
+
+      def word_doc_freq_key(word)
+        "Word:[#{word}]:doc_freq"
+      end
+
+      def word_count_key(word)
+        "Word:[#{word}]:count"
+      end
+    end
+  end
+end

data/lib/harmonizer_redis/linkage.rb

@@ -0,0 +1,188 @@
+module HarmonizerRedis
+  class Linkage < BaseObject
+    attr_reader :id
+
+    def generate_id
+      SecureRandom.uuid
+    end
+
+    def initialize(params={})
+      @content = params[:content]
+      @category_id = params[:category_id]
+    end
+
+    def save # make sure that new phrase is saved
+      # if phrase already exists : set to that phrase
+      # otherwise : create a new phrase and set linkage:phrase to that phrase
+      # linkage is also added to the category with certain id (can be used to divide tasks)
+      # Assert: all required fields have been set
+      @id = generate_id
+
+      unless @id && @content && @category_id
+        raise "id, content, and category_id are not all set"
+      end
+
+      @content_normalized = HarmonizerRedis.normalize_string(@content)
+      existing_phrase_id = HarmonizerRedis::Phrase.find_by_content(@content_normalized)
+      if existing_phrase_id
+        @phrase = existing_phrase_id
+      else
+        new_phrase = HarmonizerRedis::Phrase.new(@content_normalized)
+        new_phrase.save
+        @phrase = new_phrase.id
+      end
+      super()
+      Category.add_linkage(self)
+    end
+
+    # Readers
+
+    def content
+      @content ||= self.class.get_content(@id)
+    end
+
+    def content_normalized
+      @content_normalized ||= self.class.get_content_normalized(@id)
+    end
+
+    def category_id
+      @category_id ||= self.class.get_category_id(@id)
+    end
+
+    def corrected
+      label = Category.get_group_label(category_id, phrase_id)
+      if label.nil?
+        if Category.get_group_count(category_id, phrase_id)
+          return content
+        else
+          '(LABEL NOT SET)'
+        end
+      else
+        label
+      end
+    end
+
+    def phrase_id
+      @phrase ||= self.class.get_phrase_id(@id)
+    end
+
+    # Writers
+
+    def content=(value)
+      if self.is_saved?
+        raise "Saved linkage content cannot be edited"
+      else
+        @content = value
+      end
+    end
+
+    def category_id=(value)
+      if self.is_saved?
+        raise "Saved linkage category_id cannot be edited"
+      else
+        @category_id = value
+      end
+    end
+
+    ### Functionality
+    def calculate_similarities
+      own_phrase_id = phrase_id
+      own_cat_id = category_id
+      phrase_list = Category.get_phrase_list(own_cat_id)
+      matrix_list = Category.get_matrices(category_id, phrase_list)
+      own_matrix = Phrase.get_matrix(own_phrase_id)
+      Redis.current.pipelined do
+        phrase_list.each_with_index do |other_id, index|
+          score = Phrase.calc_soft_pair_similarity(own_matrix, matrix_list[index])
+          if score > 0.2
+            Redis.current.zadd("HarmonizerRedis::Category:#{own_cat_id}:#{own_phrase_id}:sims", score, other_id)
+          end
+        end
+      end
+      Category.set_phrase_calculated(own_cat_id, own_phrase_id, 1)
+      Category.reset_changed(own_cat_id)
+    end
+
+    def get_similarities(num_phrases = 20)
+      self_phrase_id = phrase_id
+      unless is_calculated?
+        calculate_similarities
+      end
+      phrase_id_list = Redis.current.zrevrange("HarmonizerRedis::Category:#{self.category_id}:#{self_phrase_id}:sims",
+                                               0, num_phrases, :with_scores => true)
+      results = []
+      phrase_id_list.each do |phrase, score|
+        unless Category.in_same_group?(category_id, self_phrase_id, phrase)
+          results << [Phrase.get_content(phrase), Category.get_group_label(category_id, phrase), score, phrase]
+        end
+      end
+      results
+    end
+
+    # Recommend possible labels for a linkage
+    def recommend_labels
+      existing_labels = Category.get_all_group_labels(category_id, phrase_id)
+      other_linkages = Category.get_group_popular_linkages(category_id, phrase_id)
+      existing_labels + other_linkages
+    end
+
+    def merge_with_phrase(phrase_id)
+      Category.merge_phrase_groups(category_id, self.phrase_id, phrase_id)
+    end
+
+    def set_corrected_label(label)
+      Category.set_group_label(category_id, phrase_id, label)
+    end
+
+    ### Helpers
+    def is_category_changed?
+      unless is_saved?
+        raise "Linkage must be saved first"
+      end
+      Category.changed?(self.category_id)
+    end
+
+    def is_saved?
+      self.class.is_linkage_saved?(@id)
+    end
+
+    def is_calculated?
+      if is_category_changed? || !is_saved?
+        false
+      else
+        Category.is_phrase_calculated?(category_id, phrase_id)
+      end
+    end
+
+    class << self
+      def find(linkage_id)
+        unless is_linkage_saved?(linkage_id)
+          return nil
+        end
+        linkage = self.new
+        linkage.instance_variable_set('@id', linkage_id)
+        linkage
+      end
+
+      def is_linkage_saved?(linkage_id)
+        Redis.current.sismember("#{self}:set", "#{linkage_id}")
+      end
+
+      def get_category_id(linkage_id)
+        Redis.current.get("#{self}:#{linkage_id}:category_id")
+      end
+
+      def get_phrase_id(linkage_id)
+        Redis.current.get("#{self}:#{linkage_id}:phrase")
+      end
+
+      def get_content(linkage_id)
+        Redis.current.get("#{self}:#{linkage_id}:content")
+      end
+
+      def get_content_normalized(linkage_id)
+        Redis.current.get("#{self}:#{linkage_id}:content_normalized")
+      end
+    end
+  end
+end

data/lib/harmonizer_redis/phrase.rb

@@ -0,0 +1,87 @@
+module HarmonizerRedis
+  class Phrase < BaseObject
+    attr_accessor :content
+
+    def initialize(content)
+      @content = content
+    end
+
+    def save
+      super()
+      HarmonizerRedis::IdfScorer.add_document(@id)
+      Redis.current.set("#{self.class}:[#{@content}]", "#{@id}")
+    end
+
+    class << self
+      def find_by_content(content)
+        Redis.current.get("#{self}:[#{content}]")
+      end
+
+      def get_content(phrase_id)
+        Redis.current.get("#{self}:#{phrase_id}:content")
+      end
+
+      # Setup
+      def add_linkage(phrase_id, linkage_id, category_id)
+        add_linkage_id(phrase_id, linkage_id)
+        add_category_id(phrase_id, category_id)
+      end
+
+      # Linkages
+      def add_linkage_id(phrase_id, linkage_id)
+        Redis.current.zincrby(linkage_set_key(phrase_id), 1, Linkage.get_content(linkage_id))
+      end
+
+      def get_linkage_count(phrase_id)
+        Redis.current.zcard(linkage_set_key(phrase_id))
+      end
+
+      def get_popular_linkages(phrase_id, number = 5)
+        if number <= 0
+          raise "number must be >= 0"
+        end
+        Redis.current.zrevrange(linkage_set_key(phrase_id), 0, number-1, with_scores: true)
+      end
+
+      # Categories
+      def add_category_id(phrase_id, category_id)
+        Redis.current.sadd(category_set_key(phrase_id), category_id)
+      end
+
+      def get_categories(phrase_id)
+        Redis.current.smembers(category_set_key(phrase_id))
+      end
+
+      #get a serialized version of the matrix.
+      def get_matrix(phrase_id)
+        serialized = Redis.current.get("#{self}:#{phrase_id}:matrix")
+        if serialized
+          serialized
+        else
+          nil
+        end
+      end
+
+      def calc_pair_similarity(phrase_a, phrase_b, phrase_a_matrix, phrase_b_matrix)
+        idf_similarity = IdfScorer.cos_similarity(phrase_a_matrix, phrase_b_matrix)
+        white_similarity = WhiteSimilarity.score(phrase_a, phrase_b)
+        (idf_similarity + white_similarity) * -0.5
+      end
+
+      def calc_soft_pair_similarity(phrase_a_matrix, phrase_b_matrix)
+        WhiteSimilarity.soft_cos_similarity(phrase_a_matrix, phrase_b_matrix)
+      end
+
+      ### Helpers ### (consider refactoring with metaprogramming)
+      def linkage_set_key(phrase_id)
+        "#{self}:#{phrase_id}:linkage_set"
+      end
+
+      def category_set_key(phrase_id)
+        "#{self}:#{phrase_id}:category_set"
+      end
+
+    end
+    private_class_method :linkage_set_key, :category_set_key
+  end
+end

data/lib/harmonizer_redis/version.rb

@@ -0,0 +1,3 @@
+module HarmonizerRedis
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: harmonizer_redis
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Tian Wang
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-09-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hiredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Harmonizes records based on fuzzy string/phrase matching. Built on redis
+  for speed
+email:
+- twang95@stanford.edu
+executables: []
+extensions:
+- ext/white_similarity/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ext/white_similarity/extconf.rb
+- ext/white_similarity/white_similarity.c
+- ext/white_similarity/white_similarity.h
+- harmonizer_redis.gemspec
+- lib/harmonizer_redis.rb
+- lib/harmonizer_redis/base_object.rb
+- lib/harmonizer_redis/category.rb
+- lib/harmonizer_redis/idf_scorer.rb
+- lib/harmonizer_redis/linkage.rb
+- lib/harmonizer_redis/phrase.rb
+- lib/harmonizer_redis/version.rb
+homepage: https://github.com/POSpulse/harmonizer_redis
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Harmonizes records
+test_files: []