text_profile_signature 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b40d0d688520628dfb1b3ae43e936d4c454753b9
+  data.tar.gz: 14e32db1f6b8e993a4f59740f8a28aeeeace3234
+SHA512:
+  metadata.gz: c3593a2de6b6162c85c28cb55b952ab84bd81991e10ea07634de9cecc6fbef2eb2364b1b80b948824882f3acd1ce9481f6b70bd62ca2313a1ae2ec8a5af21014
+  data.tar.gz: 5c8a4e9ec4a11208a4809cef022d22e6e6519513b7dcafad18f61695c405fd9fba7828521e7ff8d3f72122e7820cd1d514f2d22c0a5b6202be24d47f9393b9f4

data/.gitignore

@@ -0,0 +1,2 @@
+*.gem
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,15 @@
+Text Profile Signature calculates a fuzzy hash of textual fields for Deduplication.
+Copyright (C) 2016  Hamed Ramezanian Nik
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.

data/README.md

@@ -0,0 +1,55 @@
+# Text Profile Signature
+[![Gem Version](https://badge.fury.io/rb/text_profile_signature.svg)](https://badge.fury.io/rb/text_profile_signature)
+
+Text Profile Signature calculates a fuzzy hash of textual fields for Deduplication. It's the port of [TextProfileSignature](https://wiki.apache.org/solr/TextProfileSignature) which is written in Java.
+
+## Installation
+
+### RubyGems
+
+Add this to the Gemfile:
+
+    gem 'text_profile_signature'
+
+or install it directly:
+
+    gem install text_profile_signature
+
+### Install from Git
+
+Add the following in the Gemfile:
+
+    gem 'text_profile_signature', :git => 'https://github.com/iCEAGE/text_profile_signature.git'
+
+
+## Getting Started
+
+Please follow the [installation](#installation) procedure and then run the following code:
+
+```ruby
+# Load the gem
+require 'text_profile_signature'
+
+opts = {
+  :min_token_length => 2, # Default to 2
+  :quant_rate => 0.01 # Default to 0.01
+}
+
+text_profile_signature_instance = TextProfileSignature.new(opts)
+
+text = <<-STR
+Liberty, in philosophy, involves free will as contrasted with determinism.[1] In politics, liberty consists of the social and political freedoms enjoyed by all citizens.[2] In theology, liberty is freedom from the bondage of sin.[3] Generally, liberty seems to be distinct from freedom in that freedom concerns itself primarily, if not exclusively, with the ability to do as one wills and what one has the power to do; whereas liberty also takes into account the rights of all involved. As such, liberty can be thought of as freedom limited by rights, and therefore cannot be abused.
+STR
+
+sign = text_profile_signature_instance.generate_sign(text)
+
+puts sign
+
+```
+
+## Documentation for options
+
+|       Name       |  Type |                                     Description                                    | Default value |
+|:----------------:|:-----:|:----------------------------------------------------------------------------------:|---------------|
+| min_token_length |  int  | The minimum token length to consider                                               | 2             |
+|    quant_rate    | float | When multiplied by the maximum token frequency, this determines count quantization | 0.01          |

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/lib/*_test.rb']
+  t.verbose = true
+end
+
+task :default => :test

data/lib/text_profile_signature.rb

@@ -0,0 +1,106 @@
+=begin
+Text Profile Signature calculates a fuzzy hash of textual fields for Deduplication.
+Copyright (C) 2016  Hamed Ramezanian Nik
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require 'digest'
+require 'unicode'
+
+class TextProfileSignature
+  VERSION = "0.0.1" unless defined? TextProfileSignature::VERSION
+  
+  def initialize(options={})
+    options[:min_token_length] ||= 2
+    options[:quant_rate] ||= 0.01
+    
+    @options = options
+  end
+  
+  def generate_sign(text)
+    # remove all characters except letters and digits, 
+    # and bring all characters to lower case
+    # split the text into tokens (all consecutive non-whitespace characters)
+    # discard tokens equal or shorter than MIN_TOKEN_LEN (default 2 characters)
+    current_token = String.new
+    max_freq = 0
+    tokens = {}
+    text.each_char do |character|
+      if character =~ /[[:alnum:]]/
+        current_token << Unicode::downcase(character)
+      else
+        if current_token.length > 0
+          if current_token.length > @options[:min_token_length]
+            # Add it
+            tok = tokens[current_token]
+            unless tok
+              tok = {count: 0, term: current_token}
+              tokens[current_token] = tok
+            end
+            tok[:count] += 1
+            max_freq = tok[:count] if tok[:count] > max_freq
+          end
+          current_token = String.new
+        end
+      end
+    end
+    
+    # Check the last token
+    if current_token.length > @options[:min_token_length]
+      # Add it
+      tok = tokens[current_token]
+      unless tok
+        tok = {count: 0, term: current_token}
+        tokens[current_token] = tok
+      end
+      tok[:count] += 1
+      max_freq = tok[:count] if tok[:count] > max_freq
+    end
+    
+    # calculate the QUANT value
+    quant = (max_freq * @options[:quant_rate]).round
+    
+    if quant < 2
+      if max_freq > 1
+        quant = 2
+      else
+        quant = 1
+      end
+    end
+    
+    # round down the counts of tokens to the nearest multiple of QUANT
+    # tokens, which frequency after quantization falls below QUANT, are discarded
+    quantized_tokens = tokens.values.inject([]) do |memo, item|
+      # round down to the nearest QUANT
+      item[:count] = (item[:count] / quant) * quant
+      
+      # discard the frequencies below the QUANT
+      memo.push(item) if item[:count] >= quant
+      
+      memo
+    end
+    
+    # sort the list of tokens by decreasing frequency
+    profile = quantized_tokens.sort {|x, y| y[:count] <=> x[:count]}
+
+    # create a list of tokens and their quantized frequency, 
+    # separated by spaces, in the order of decreasing frequency
+    quantized_frequency_str = profile.map do |a|
+      "#{a[:term]} #{a[:count]}"
+    end.join('\n')
+    
+    Digest::MD5.hexdigest(quantized_frequency_str)
+  end
+end

data/test/fixtures/liberty_article_from_de_wikipedia.yaml

@@ -0,0 +1,7 @@
+:signature: 6a357987bbb275181328aec84aab0e00
+:article: |
+  Freiheit (lateinisch libertas) wird in der Regel verstanden als die Möglichkeit, ohne Zwang zwischen unterschiedlichen Möglichkeiten auswählen und entscheiden zu können. Der Begriff benennt in Philosophie, Theologie und Recht der Moderne allgemein einen Zustand der Autonomie eines Subjekts.
+  
+  Ebenfalls von rechtlicher, politischer und philosophischer Bedeutung ist die Unterscheidung zwischen positiver und negativer Freiheit, die sich nur zum Teil mit der Unterscheidung von inneren und äußern Beschränkungen der Handlungsfreiheit deckt. Sie ist vor allem sozialphilosophisch aufgeladen.[5] Die Unterscheidung findet sich schon bei Aristoteles, sie ist aber über die Tradition von Thomas Hobbes und Immanuel Kant zentrales Element des Liberalismus auch im 20. Jahrhundert geworden, dessen Hauptanliegen politische Selbstbestimmung, Schutz des Individuums und Freiheit des Wirtschaftshandelns (als Voraussetzung eines allgemeinen Wohlstandszuwachses und einer daraus resultierenden erweiterten Handlungsfähigkeit) sind. Negative Freiheit (Freiheit von etwas) bezeichnet einen Zustand, in dem keine von der Regierung, der Gesellschaft oder anderen Menschen ausgehenden Zwänge ein Verhalten erschweren oder verhindern;[6][7] Positive Freiheit (Freiheit zu etwas) bezeichnet die Möglichkeit der Selbstverwirklichung, insbesondere der demokratischen Selbstregierung einer Gemeinschaft.[8] Einige Sozialwissenstheoretiker, wie Ralf Dahrendorf lehnen diese Begriffe von Freiheit ab und vertreten stattdessen das Konzept einer einzigen sozialen Freiheit. Diese wird definiert als Abwesenheit externer sozialer Beschränkungen und dem Vorhandensein zumindest eines notwendigen Minimums an sozialen Handlungsressourcen.[9]
+  
+  Im Allgemeinen wird auch bürgerlich-rechtlich die positive Freiheit von der negativen unterschieden. Die positive Freiheit (nicht zu verwechseln mit dem Positivismus) meint die Freiheit zu etwas, bspw. das Recht des Bürgers auf Bewegungsfreiheit oder Meinungsfreiheit. Negative Freiheit hingegen bezeichnet die Freiheit von etwas, bspw. von staatlicher Intervention im persönlichem oder künstlerischem Bereich.[10]

data/test/fixtures/liberty_article_from_en_wikipedia.yaml

@@ -0,0 +1,15 @@
+:signature: f18022e6b522fa7249ab73498728c275
+:article: |
+  Liberty, in philosophy, involves free will as contrasted with determinism.[1] In politics, liberty consists of the social and political freedoms enjoyed by all citizens.[2] In theology, liberty is freedom from the bondage of sin.[3] Generally, liberty seems to be distinct from freedom in that freedom concerns itself primarily, if not exclusively, with the ability to do as one wills and what one has the power to do; whereas liberty also takes into account the rights of all involved. As such, liberty can be thought of as freedom limited by rights, and therefore cannot be abused.
+
+
+
+  Philosophers from earliest times have considered the question of liberty. Roman Emperor Marcus Aurelius (121–180 AD) wrote of "a polity in which there is the same law for all, a polity administered with regard to equal rights and equal freedom of speech, and the idea of a kingly government which respects most of all the freedom of the governed."[4] According to Thomas Hobbes, "a free man is he that in those things which by his strength and wit he is able to do is not hindered to do what he hath the will to do" (Leviathan, Part 2, Ch. XXI).
+
+  John Locke (1632–1704) rejected that definition of liberty. While not specifically mentioning Hobbes, he attacks Sir Robert Filmer who had the same definition. According to Locke:
+
+      "In the state of nature, liberty consists of being free from any superior power on Earth. People are not under the will or lawmaking authority of others but have only the law of nature for their rule. In political society, liberty consists of being under no other lawmaking power except that established by consent in the commonwealth. People are free from the dominion of any will or legal restraint apart from that enacted by their own constituted lawmaking power according to the trust put in it. Thus, freedom is not as Sir Robert Filmer defines it: 'A liberty for everyone to do what he likes, to live as he pleases, and not to be tied by any laws.' Freedom is constrained by laws in both the state of nature and political society. Freedom of nature is to be under no other restraint but the law of nature. Freedom of people under government is to be under no restraint apart from standing rules to live by that are common to everyone in the society and made by the lawmaking power established in it. Persons have a right or liberty to (1) follow their own will in all things that the law has not prohibited and (2) not be subject to the inconstant, uncertain, unknown, and arbitrary wills of others."[5]
+
+  John Stuart Mill.
+
+  John Stuart Mill (1806–1873), in his work, On Liberty, was the first to recognize the difference between liberty as the freedom to act and liberty as the absence of coercion.[6] In his book, Two Concepts of Liberty, Isaiah Berlin formally framed the differences between these two perspectives as the distinction between two opposite concepts of liberty: positive liberty and negative liberty. The latter designates a negative condition in which an individual is protected from tyranny and the arbitrary exercise of authority, while the former refers to the liberty that comes from self-mastery, the freedom from inner compulsions such as weakness and fear.

data/test/fixtures/liberty_article_from_es_wikipedia.yaml

@@ -0,0 +1,19 @@
+:signature: 8cc2cf890f4632b55899a3184b7b4a96
+:article: |
+  La libertad (del latín: libertas, -ātis)1 es la capacidad de la conciencia para pensar y obrar según la propia voluntad de la persona pero en sujeción a un orden o regulación más elevados.
+  
+  Según las acepciones 1, 2, 3 y 4 de este término en el diccionario de la RAE,2 el estado de libertad define la situación, circunstancias o condiciones de quien no es esclavo, ni sujeto, ni impuesto al deseo de otros de forma coercitiva. En otras palabras, aquello que permite a alguien decidir si quiere hacer algo o no, lo hace libre, pero también responsable de sus actos en la medida en que comprenda las consecuencias de ellos. Pues la libertad implica una clara opción por el bien y el mal, solo desde esta opción se estaría actuando desde la concepción de la Teleología[cita requerida].
+  
+  La quinta acepción del término2 define la libertad como la "facultad que se disfruta en las naciones bien gobernadas de hacer y decir cuanto no se oponga a las leyes ni a las buenas costumbres". Con base a ello, la protección de la libertad interpersonal, es objeto de una investigación social y política.
+  
+  El fundamento metafísico de la libertad interior es una cuestión psicológica y filosófica. Ambas formas de la libertad se unen en cada individuo como lo interno y lo externo de una malla de valores, juntos en una dinámica de compromiso.
+  
+  En castellano la palabra libertad proviene del latín libertas, -ātis, de igual significado. La palabra inglesa para libertad, freedom, proviene de una raíz indoeuropea que significa "amar"; la palabra de la misma lengua para decir miedo, afraid, viene de la misma raíz, usado como contraposición a libertad mediante el prefijo a por influencia del latín vulgar.
+  
+  La libertad como desaparición de opresión significa no querer subyugar ni ser subyugado, e implica el fin de un estado de servidumbre. El logro de esta forma de la libertad depende de una combinación de la resistencia del individuo (o grupo) y su entorno.
+  
+  Las leyes artificiales limitan esta forma de libertad, por ejemplo, nadie es libre de no ser representado por políticos dentro de una nación (aunque podamos o no ser libres para intentarlo).
+  
+  Las leyes naturales, como las leyes físicas, o la ley de la gravedad, son también un fundamento importante para la libertad de todos los seres vivos existentes en el universo.
+  
+  Todos los actos presuponen a la libertad para poder ser moralmente imputables (libre albedrío). La libertad se sitúa en la interioridad de la persona y siguiendo esa línea de pensamiento afirma Ricardo Yepes Stork: "Es una de las notas definitorias de la persona. Permite al hombre alcanzar su máxima grandeza pero también su mayor degradación. Es quizás su don más valioso porque empapa y define todo su actuar. El hombre es libre desde lo más profundo de su ser. Por eso los hombres modernos han identificado el ejercicio de la libertad con la realización de la persona: se trata de un derecho y de un ideal al que no podemos ni queremos renunciar. No se concibe que se pueda ser verdaderamente humano sin ser libre de verdad."[cita requerida]

data/test/fixtures/liberty_article_from_fr_wikipedia.yaml

@@ -0,0 +1,16 @@
+:signature: 728ce35eacc5e3ac743d6931f422caf0
+:article: |
+  De façon générale, la liberté est un concept qui désigne la possibilité d'action ou de mouvement.
+   En mécanique par exemple, on parle de degrés de liberté pour comptabiliser les mouvements possibles d'une pièce.
+  
+  Pour le sens commun, la liberté s'oppose à la notion d'enfermement ou de séquestration. Une personne qui vient de sortir de prison est dite libre. Le sens originel du mot liberté est d'ailleurs assez proche : l'homme libre est celui qui n'appartient pas à autrui, qui n'a pas le statut d'esclave.
+  
+  En philosophie, en sociologie, en droit et en politique, la liberté est une notion majeure : elle marque l'aptitude des individus à exercer leur volonté avec — selon l'orientation politique des discours tenus — la mise en avant de nuances dont aucune n'épuise le sens intégral :
+  
+  formulation négative : où l'on pointe l'absence de soumission, de servitude, de contrainte, qu'elles soient exercées par d'autres individus (comme pour l'esclavage), ou par la société (c'est-à-dire par la Loi).
+  
+  formulation positive : où l'on affirme l'autonomie et la spontanéité du sujet rationnel ; les comportements humains volontaires se fondent sur la liberté et sont qualifiés de libres.
+  
+  formulation relative : différents adages font ressortir l'équilibre à trouver dans une alternative, visant notamment à rendre la liberté compatible avec des principes de philosophie politique tels que l'égalité et la justice. Ainsi : La « liberté consiste à pouvoir faire tout ce qui ne nuit pas à autrui » (art. 4 de la Déclaration des droits de l'homme), ce qui implique la possibilité de « faire tout ce qui n'est point interdit, comme ne pas faire ce qui n'est point obligatoire » (art. 5), la « liberté de dire ou de faire ce qui n'est pas contraire à l'ordre public ou à la morale publique » (droit administratif) ou encore « La liberté des uns s'arrête là où commence celle des autres » (peut-être inspiré par John Stuart Mill)1. Dans une telle formulation, la liberté est étroitement liée au concept de droit, allant jusqu'à confondre les deux notions.
+  
+  Dans la mesure où ces deux perspectives se recoupent de diverses manières, leur chevauchement peut provoquer des erreurs d'interprétation dans les analyses et la confusion dans les débats. Il faut donc prendre soin de distinguer les différents sens de ce mot.

data/test/fixtures/liberty_article_from_it_wikipedia.yaml

@@ -0,0 +1,13 @@
+:signature: 4c8484da0faa43e345d956e13a4a61f4
+:article: |
+  Per libertà s'intende la condizione per cui un individuo può decidere di pensare, esprimersi ed agire senza costrizioni, ricorrendo alla volontà di ideare e mettere in atto un'azione, mediante una libera scelta dei fini e degli strumenti che ritiene utili a realizzarla.
+  
+  Secondo una concezione non solo kantiana, la libertà è una condizione formale della scelta che, quando si tramuterà in atto, in azione concreta, risentirà necessariamente dei condizionamenti che le vengono dal mondo reale, sottoposto alle leggi fisiche necessitanti, o da situazioni determinanti di altra natura.
+  
+  Riguardo all'ambito in cui si opera la libera scelta si parla di libertà morale, giuridica, economica, politica, di pensiero, libertà metafisica, religiosa ecc.
+  
+   «L'essenza della libertà è sempre consistita nella capacità di scegliere come si vuole scegliere e perché così si vuole, senza costrizioni o intimidazioni, senza che un sistema immenso ci inghiotta; e nel diritto di resistere, di essere impopolare, di schierarti per le tue convinzioni per il solo fatto che sono tue. La vera libertà è questa, e senza di essa non c'è mai libertà, di nessun genere, e nemmeno l'illusione di averla»[1]
+
+  La mitologia romana, che pure aveva tratto da quella greca molte divinità e miti, ne possedeva alcuni che appartenevano solo ai loro riti come quello della dea Libertà che rappresentava simbolicamente la libertà personale di ognuno e, nel seguito della loro storia civile, il diritto riservato a coloro che godevano della cittadinanza romana.
+  
+  A questa divinità i Romani avevano innalzato due templi, uno nel Foro e l'altro nell'Aventino. La dea veniva raffigurata come una donna, con ai piedi un gatto, recante in una mano uno scettro e nell'altra mano un berretto frigio.[2][3]

data/test/fixtures/liberty_article_from_pt_wikipedia.yaml

@@ -0,0 +1,21 @@
+:signature: f18022e6b522fa7249ab73498728c275
+:article: |
+  Liberdade, em filosofia, pode ser compreendida sob uma perspectiva que denota a ausência de submissão e de servidão. Ou sob outra perspectiva que é a autonomia e a espontaneidade de um sujeito racional.
+  
+  Para o filósofo René Descartes (1596-1650), age com mais liberdade quem melhor compreende as alternativas que precedem a escolha. Dessa premissa, decorre o silogismo lógico de que, quanto mais evidente a veracidade de uma alternativa, maiores as chances de ela ser escolhida pelo agente. Nesse sentido, a inexistência de acesso à informação afigura-se óbice à identificação da alternativa com maior grau de veracidade.
+  
+  Para Baruch Espinoza (1632-1677), a liberdade possui um elemento de identificação com a natureza do "ser". Nesse sentido, ser livre significa agir de acordo com sua natureza.
+  
+  É mediante a liberdade que o Homem se exprime como tal e em sua totalidade. Esta é também, enquanto meta dos seus esforços, a sua própria realização.
+  
+  Tendemos a associar a fruição da liberdade a uma determinação constante e inescapável. Contudo, os ditames de nossa vida estão sendo realizados a cada passo que damos: assim, a deliberação está também a cargo da vontade humana (na qual se inserem as leis físicas e químicas, biológicas e psicológicas).
+  
+  Diretamente associada à ideia de liberdade, está a noção de responsabilidade, vez que o ato de ser livre implica assumir o conjunto dos nossos atos e saber responder por eles.
+  
+  Para Arthur Schopenhauer (1788-1860), a ação humana não é absolutamente livre. Todo o agir humano, bem como todos os fenômenos da natureza, até mesmo suas leis, são níveis de objetivação da coisa-em-si kantiana que o filósofo identifica como sendo puramente Vontade.
+  
+  Para Schopenhauer, o homem é capaz de acessar sua realidade por um duplo registro: o primeiro, o do fenômeno, onde todo o existente reduz-se, nesse nível, a mera representação.
+  
+  No nível essencial, que não deixa-se apreender pela intuição intelectual, pela experiência dos sentidos, o mundo é apreendido imediatamente como vontade, Vontade de Vida. Nesse caso, a noção de vontade assume um aspecto amplo e aberto, transformando-se no princípio motor dos eventos que sucedem-se na dimensão fenomênica segundo a lei da causalidade.
+  
+  O homem, objeto entre objetos, coisa entre coisas, não possui liberdade de ação porque não é livre para deliberar sobre sua vontade. O homem não escolhe o que deseja, o que quer. Logo, não é livre - é absolutamente determinado a agir segundo sua vontade particular, objetivação da vontade metafísica por trás de todos os eventos naturais. O que parece deliberação é uma ilusão ocasionada pela mera consciência sobre os próprios desejos.É poder viver sem ninguém mandar.

data/test/lib/text_profile_signature_unit_test.rb

@@ -0,0 +1,63 @@
+=begin
+Text Profile Signature calculates a fuzzy hash of textual fields for Deduplication.
+Copyright (C) 2016  Hamed Ramezanian Nik
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..'))
+
+require 'test_helper'
+
+class TextProfileSignatureGeneratorTest < TextProfileSignatureTest
+  def setup
+    @text_profile_signature = TextProfileSignature.new
+  end
+  
+  def test_en_lang
+    page = get_wikipedia_article("en")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+  
+  def test_de_lang
+    page = get_wikipedia_article("de")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+  
+  def test_es_lang
+    page = get_wikipedia_article("es")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+  
+  def test_fr_lang
+    page = get_wikipedia_article("fr")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+  
+  def test_it_lang
+    page = get_wikipedia_article("it")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+  
+  def test_pt_lang
+    page = get_wikipedia_article("it")
+    sign = @text_profile_signature.generate_sign(page[:article])
+    assert_equal sign, page[:signature]
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,29 @@
+=begin
+Text Profile Signature calculates a fuzzy hash of textual fields for Deduplication.
+Copyright (C) 2016  Hamed Ramezanian Nik
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Lesser General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Lesser General Public License for more details.
+
+You should have received a copy of the GNU Lesser General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+=end
+
+require File.expand_path('../../lib/text_profile_signature', __FILE__)
+
+require 'minitest/autorun'
+require 'yaml'
+
+class TextProfileSignatureTest < MiniTest::Test
+  def get_wikipedia_article(lang)
+    path = File.expand_path("../fixtures/liberty_article_from_#{lang}_wikipedia.yaml", __FILE__)
+    YAML.load_file(path)
+  end
+end

data/text_profile_signature.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/text_profile_signature', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "text_profile_signature"
+  gem.version       = TextProfileSignature::VERSION
+  gem.platform      = Gem::Platform::RUBY
+  gem.authors       = ["Hamed Ramezanian Nik"]
+  gem.email         = ["hamed.r.nik@gmail.com"]
+  gem.summary       = "A fuzzy hash of text generator for Deduplication"
+  gem.description   = "A fuzzy hash of text generator for Deduplication."
+  gem.homepage      = "https://github.com/iCEAGE/text_profile_signature"
+  gem.license       = "LGPL-3.0"
+
+  gem.files         = `git ls-files | grep -Ev '^(myapp|examples)'`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+  
+  gem.add_runtime_dependency  'unicode', '~> 0.4.4.2'
+
+  gem.add_development_dependency 'rake', '~> 11.1', '>= 11.1.2'
+  gem.add_development_dependency 'minitest', '~> 5.9'
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: text_profile_signature
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hamed Ramezanian Nik
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-06-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: unicode
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.4.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.1.2
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.9'
+description: A fuzzy hash of text generator for Deduplication.
+email:
+- hamed.r.nik@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/text_profile_signature.rb
+- test/fixtures/liberty_article_from_de_wikipedia.yaml
+- test/fixtures/liberty_article_from_en_wikipedia.yaml
+- test/fixtures/liberty_article_from_es_wikipedia.yaml
+- test/fixtures/liberty_article_from_fr_wikipedia.yaml
+- test/fixtures/liberty_article_from_it_wikipedia.yaml
+- test/fixtures/liberty_article_from_pt_wikipedia.yaml
+- test/lib/text_profile_signature_unit_test.rb
+- test/test_helper.rb
+- text_profile_signature.gemspec
+homepage: https://github.com/iCEAGE/text_profile_signature
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.4
+signing_key: 
+specification_version: 4
+summary: A fuzzy hash of text generator for Deduplication
+test_files:
+- test/fixtures/liberty_article_from_de_wikipedia.yaml
+- test/fixtures/liberty_article_from_en_wikipedia.yaml
+- test/fixtures/liberty_article_from_es_wikipedia.yaml
+- test/fixtures/liberty_article_from_fr_wikipedia.yaml
+- test/fixtures/liberty_article_from_it_wikipedia.yaml
+- test/fixtures/liberty_article_from_pt_wikipedia.yaml
+- test/lib/text_profile_signature_unit_test.rb
+- test/test_helper.rb