rpareia 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8fb7c8a13b2133209c080eaf3eba626412e71659
+  data.tar.gz: e3acad44a34d0ba4d74b746aa82fb09cce9ddf64
+SHA512:
+  metadata.gz: 0ab1079409ef2541c80d838925b59b8f8f11d259e993248c326e9f451091906184299870bc14c9364be02b68175f8a85c4b66091a4d81d2bc4a692efbad57d9b
+  data.tar.gz: 1a5182f13b2f71fe6ecac5cfe07b6488b789da43c583b9e5245ab84a1e40dec67254a2721f46240faee4ef8addb70974e2c86e3537b99b6b24f8c48784cdf6b0

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--format doc

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - "2.1.0"
+  - "2.2.0"

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rpareia.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,84 @@
+PATH
+  remote: .
+  specs:
+    rpareia (0.0.1)
+      gli (~> 2.12)
+      nokogiri (~> 1.6.6)
+      rbczmq (~> 1.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.6)
+    formatador (0.2.5)
+    gli (2.12.2)
+    guard (2.11.1)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.5.0)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    hitimes (1.2.2)
+    listen (2.8.5)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.9)
+    method_source (0.8.2)
+    mini_portile (0.6.2)
+    nenv (0.2.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    notiffany (0.0.3)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.4.2)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    rbczmq (1.7.8)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.0)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.1)
+    shellany (0.0.1)
+    slop (3.6.0)
+    thor (0.19.1)
+    timers (4.0.1)
+      hitimes
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.7)
+  guard-rspec (~> 4.5)
+  pry (~> 0.10)
+  rake (~> 10.0)
+  rpareia!
+  rspec (~> 3.2)

data/Guardfile

@@ -0,0 +1,5 @@
+guard :rspec, cmd: "bundle exec rspec" do
+  watch(%r{^spec/(.+)/.+_spec\.rb$})
+  watch(%r{^lib/(.+)/(.+)\.rb$}) { |m| "spec/#{m[1]}/#{m[2]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Michel Boaventura
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,24 @@
+[![CodeClimate](https://codeclimate.com/github/michelboaventura/rpareia/badges/gpa.svg)](https://codeclimate.com/github/michelboaventura/rpareia)
+[![Travis](https://travis-ci.org/michelboaventura/rpareia.svg)](https://travis-ci.org/michelboaventura/rpareia.svg)
+
+# Rpareia
+
+Pareia's[1] implementation in Ruby with zeromq:
+
+[1] https://github.com/michelboaventura/pareia
+
+## Installation
+
+    $ gem install rpareia
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/michelboaventura/rpareia/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+
+begin
+  require "rspec/core/rake_task"
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+  # no rspec available
+  # end
+end

data/bin/rpareia

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'gli'
+require 'rpareia'
+
+include GLI::App
+
+program_desc "A data deduplication software"
+
+desc "Handles with project files"
+command :project do |c|
+  c.desc 'Check a project xml for errors'
+  c.flag [:c,:check]
+
+  c.action do |global_options,options,args|
+    if project = options[:c]
+      Rpareia::Parser.new(File.open(project,'r').read)
+      puts "Project file '#{project}' is OK"
+    end
+  end
+end
+
+exit run(ARGV)

data/lib/rpareia.rb

@@ -0,0 +1,6 @@
+require "rpareia/version"
+require "rpareia/parser"
+
+module Rpareia
+  # Your code goes here...
+end

data/lib/rpareia/blocker.rb

@@ -0,0 +1,52 @@
+require_relative 'heartbeat'
+require 'rbczmq'
+require 'pry'
+
+class Blocker
+
+  BLOCKER_PORT_START = 2000
+  CTX = ZMQ::Context.new
+
+  def initialize(rank:, debug: false)
+    @rank = rank
+    @debug = debug
+    @port = BLOCKER_PORT_START + @rank
+  end
+
+  def start
+    start_hb
+    connect
+
+    total = 0
+
+    loop do
+      msg = @socket.recv
+
+      next if msg.nil? || msg[0] == "\x00"
+
+      if msg != "EOF"
+        total += 1
+      else
+        puts "TOTAL=#{total}"
+      end
+    end
+
+  end
+
+  def start_hb
+    Thread.new do
+      loop do
+        Heartbeat.check(addr: "tcp://*:#{2 * @port}", ctx: CTX, type: 'server', debug: @debug)
+      end
+    end
+  end
+
+  def connect
+    @socket = CTX.socket(:ROUTER)
+    @socket.verbose = @debug
+    @socket.bind("tcp://*:#{@port}")
+
+  end
+end
+
+Blocker.new(rank: ARGV[0].to_i).start

data/lib/rpareia/heartbeat.rb

@@ -0,0 +1,17 @@
+class Heartbeat
+  def self.check(addr:, ctx:, type:, debug: false)
+    if(type == 'client')
+      hb = ctx.socket(:REQ)
+      hb.verbose = debug
+      hb.connect(addr)
+      hb.send("")
+      hb.close
+    else
+      hb = ctx.socket(:REP)
+      hb.verbose = debug
+      hb.bind(addr)
+      hb.recv
+      hb.close
+    end
+  end
+end

data/lib/rpareia/parser.rb

@@ -0,0 +1,171 @@
+require "bundler/setup"
+require "nokogiri"
+require "pry"
+
+module Rpareia
+  class Parser
+    attr_reader :project
+
+    class SyntaxError < StandardError; end
+    class InvalidTaskError < StandardError; end
+    class InvalidNumberOfSources < StandardError; end
+    class DuplicatedDataSourceId < StandardError; end
+    class MissingDataSourceId < StandardError; end
+    class MissingDataSourceFile < StandardError; end
+    class InvalidDataSourceFile < StandardError; end
+    class InvalidDataSourceType < StandardError; end
+    class MissingFieldSeparator < StandardError; end
+    class FieldsElementNotFound < StandardError; end
+    class MissingFieldName < StandardError; end
+    class InvalidFieldType < StandardError; end
+    class DuplicatedFieldName < StandardError; end
+    class DeterministicLinkageElementNotFound < StandardError; end
+    class ConjunctionElementNotFound < StandardError; end
+    class MultipleConjunctionElements < StandardError; end
+    class MissingPart < StandardError; end
+    class MissingFieldName < StandardError; end
+    class MissingPartFieldNameOnDataSource < StandardError; end
+    class MissingOutputElement < StandardError; end
+    class MultipleOutputElement < StandardError; end
+    class MissingDeterministicAttribute < StandardError; end
+
+    def initialize(xml)
+      @xml = xml
+
+      parse
+    end
+
+    private
+
+    def parse_xml
+      @xml = Nokogiri::XML(@xml)
+
+      raise SyntaxError.exception(@xml.errors.join("\n")) unless @xml.errors.empty?
+    end
+
+    def parse_task
+      @project = {name: @xml.xpath("/project/@task").first.value}
+    end
+
+    def find_duplicated(arr)
+      arr.detect {|e| arr.rindex(e) != arr.index(e) }
+    end
+
+    def parse_data_sources
+      @project[:data_sources] = []
+
+      @xml.xpath("/project/data-sources/data-source").each do |data_source|
+        id = data_source['id'].to_s
+        raise MissingDataSourceId.exception if id.empty?
+
+        file = data_source['file'].to_s
+        raise MissingDataSourceFile.exception("Missing file attribute from data source '#{id}'") if file.empty?
+        raise InvalidDataSourceFile.exception("File '#{file}' from data source '#{id}' does not exist") unless File.exist?(file)
+
+        type = data_source['type'].to_s
+        raise InvalidDataSourceType.exception("Data source type '#{type}' not supported") if type != "delimited"
+
+        field_separator = data_source['field-separator'].to_s
+        raise MissingFieldSeparator.exception("Missing field separator from data source '#{id}'") if field_separator.empty?
+
+        fields = data_source.xpath("fields/field")
+        raise FieldsElementNotFound.exception("Element 'field' not found on data source '#{id}'") if fields.empty?
+
+        my_fields = []
+        fields.each do |field|
+          name = field['name']
+          raise MissingFieldName.exception("Attribute 'name' not found on data source '#{id}'") if name.empty?
+
+          type = field['type']
+          raise InvalidFieldType.exception("Invalid type '#{type}' from field '#{name}', data source '#{id}'") unless ['int', 'string'].include? type
+          my_fields << {name: name, type: type}
+        end
+
+        if dup = find_duplicated(my_fields.map{|f| f[:name]})
+          raise DuplicatedFieldName.exception("Duplicated field name '#{dup}' on data source '#{id}'")
+        end
+
+        @project[:data_sources] << {
+          id: id,
+          file: file,
+          fields: my_fields
+        }
+      end
+
+      if dup = find_duplicated(@project[:data_sources].map{|el| el[:id]})
+        raise DuplicatedDataSourceId.exception("Duplicated data source id '#{dup}'")
+      end
+
+      case @project[:name]
+      when 'linkage'
+        if @project[:data_sources].size != 2
+          raise InvalidNumberOfSources.exception("Linkage: expected two data-source, #{@project[:data_sources].size} given")
+        end
+      when 'deduplication'
+        if @project[:data_sources].size != 1
+          raise InvalidNumberOfSources.exception("Deduplication: expected one data-sources, #{@project[:data_sources].size} given")
+        end
+      else
+        raise InvalidTaskError.exception("Invalid task: '#{@project[:name]}'")
+      end
+    end
+
+    def parse_deterministic_linkage
+      deterministic_linkage = @xml.xpath("/project/deterministic-linkage")
+      raise DeterministicLinkageElementNotFound.exception("Missing deterministic-linkage element") if deterministic_linkage.empty?
+
+      conjunction = deterministic_linkage.xpath("conjunction")
+      size = conjunction.size
+
+      case size
+      when 0
+        raise ConjunctionElementNotFound.exception("Missing conjunction element")
+      when 1
+      else
+        raise MultipleConjunctionElements.exception("Only one conjunction element is allowed, #{size} found")
+      end
+
+      parts = conjunction.xpath("part")
+      raise MissingPart.exception("At leas one part element is required") if parts.empty?
+
+      @project[:parts] = []
+      parts.each do |part|
+        field_name = part['field-name'].to_s
+        raise MissingFieldName.exception("Missing attribute field-name on part element") if field_name.empty?
+
+        @project[:parts] << {field_name: field_name}
+      end
+
+      @project[:parts].each do |part|
+        @project[:data_sources].each do |data_source|
+          unless data_source[:fields].map{|e| e[:name]}.include?(part[:field_name])
+            raise MissingPartFieldNameOnDataSource.exception("Field name '#{part[:field_name]}' not found on data source '#{data_source[:id]}'")
+          end
+        end
+      end
+    end
+
+    def parse_output
+      output = @xml.xpath("/project/output")
+
+      case size = output.size
+      when 0
+        raise MissingOutputElement.exception("Missing output element")
+      when 1
+      else
+        raise MultipleOutputElement.exception("Only one output element is allowed, #{size} found")
+      end
+
+      @project[:output] = output.first['deterministic']
+      raise MissingDeterministicAttribute.exception("Missing attribute 'deterministic' on output element") unless @project[:output]
+    end
+
+    def parse
+      parse_xml
+      parse_task
+      parse_data_sources
+      parse_deterministic_linkage
+      parse_output
+    end
+  end
+end

data/lib/rpareia/reader.rb

@@ -0,0 +1,91 @@
+require 'bundler/setup'
+require 'rbczmq'
+require_relative 'heartbeat'
+require 'pry'
+
+class Reader
+  BLOCKER_PORT_START = 2000
+  CTX = ZMQ::Context.new
+
+  def initialize(num_readers:, num_blocks:, rank:, file:, debug: false, addr: "tcp://localhost")
+    @num_readers = num_readers
+    @num_blocks = num_blocks
+    @rank = rank
+    @file = file
+    @debug = debug
+    @addr = addr
+  end
+
+  def start
+    start_connections
+    read_database
+  end
+
+  def start_connections
+
+    @connections = []
+
+    0.upto(@num_blocks - 1) do |i|
+      port = BLOCKER_PORT_START + i
+      Heartbeat.check(addr: "#{@addr}:#{2 * port}", ctx: CTX, type: 'client', debug: @debug)
+
+      @connections << create_socket(port)
+    end
+  end
+
+  def create_socket(port)
+    socket = CTX.socket(:DEALER)
+    socket.verbose = @debug
+    socket.connect("#{@addr}:#{port}")
+    return socket
+  end
+
+  def read_database
+    i = -1
+    File.open(@file,'r').each_line do |line|
+      i += 1
+      next if i % @num_readers != @rank
+
+      pair = create_pair(line.strip)
+      send_pair(pair)
+    end
+    close_connections
+    puts "EOF"
+  end
+
+  def close_connections
+    @connections.each do |c|
+      c.send("EOF")
+      c.close
+    end
+    CTX.destroy
+  end
+
+  def send_pair(pair)
+    block_id = which_block(pair.last)
+
+    @connections[block_id].send(pair.join(','))
+  end
+
+  def create_hash(input)
+    input.join
+  end
+
+  def which_block(key)
+    key.bytes.first % @num_blocks
+  end
+
+  def create_pair(input)
+    line = input.split(',')
+    [line.first, create_hash(line[1..-1])]
+  end
+end
+
+Reader.new(
+  debug: false,
+  num_readers: 2,
+  num_blocks: 1,
+  rank: ARGV[0].to_i,
+  file: (ARGV[1] || "input/10m.csv"),
+  addr: (ARGV[2] || "tcp://localhost")
+).start

data/lib/rpareia/version.rb

@@ -0,0 +1,3 @@
+module Rpareia
+  VERSION = "0.0.1"
+end

data/rpareia.gemspec

@@ -0,0 +1,28 @@
+require File.join([File.dirname(__FILE__),'lib','rpareia','version.rb'])
+
+Gem::Specification.new do |s|
+  s.name          = "rpareia"
+  s.description   = "A data deduplication software, based on Pareia"
+  s.version       = Rpareia::VERSION
+  s.authors       = ["Michel Boaventura"]
+  s.email         = ["michel.boaventura@gmail.com"]
+  s.homepage      = "http://github.com/michelboaventura/rpareia"
+  s.platform      = Gem::Platform::RUBY
+  s.summary       = %q{Pareia implementation, with ruby and ZeroMQ}
+  s.files         = `git ls-files -z`.split("\x0")
+  s.license       = "MIT"
+  s.require_paths = ["lib"]
+  s.bindir        = 'bin'
+  s.executables   = ['rpareia']
+  s.test_files    = s.files.grep(%r{^(spec)/})
+
+  s.add_development_dependency "bundler", "~> 1.7"
+  s.add_development_dependency "rake", "~> 10.0"
+  s.add_development_dependency "pry", "~> 0.10"
+  s.add_development_dependency "rspec", "~> 3.2"
+  s.add_development_dependency "guard-rspec", "~> 4.5"
+
+  s.add_dependency "nokogiri", "~> 1.6"
+  s.add_dependency "rbczmq", "~> 1.7"
+  s.add_dependency "gli", "~> 2.12"
+end

data/spec/rpareia/parser_spec.rb

@@ -0,0 +1,215 @@
+include Rpareia
+
+DEFAULT_FILE = "spec/fixtures/empty_file.xml"
+
+def xml_dedup(task: 'deduplication', ids: [0], files: [DEFAULT_FILE], types: ['delimited'], field_separators: ["\t"], fields: [[{name: 'id', type: 'int'}]], parts: [{:'field-name' => 'id'}], deterministic: true, conjunction: 1, output_deterministic: ['foo'])
+  xml(task: task, ids: ids, files: files, types: types, field_separators: field_separators, fields: fields, parts: parts, deterministic: deterministic, conjunction: conjunction, output_deterministic: output_deterministic)
+end
+
+def xml_linkage(task: 'linkage', ids: [0,1], files: [DEFAULT_FILE, DEFAULT_FILE], types: ['delimited', 'delimited'], field_separators: ["\t", "\t"], fields: [[{name: 'id', type: 'int'}], [{name: 'id', type: 'int'}]], parts: [{:'field-name' => 'id'}], deterministic: true, conjunction: 1, output_deterministic: ['foo'])
+
+  xml(task: task, ids: ids, files: files, types: types, field_separators: field_separators, fields: fields, parts: parts, deterministic: deterministic, conjunction: conjunction, output_deterministic: output_deterministic)
+end
+
+def xml(task:, ids: [], files: nil, types: nil, field_separators: nil, fields: nil, parts: nil, deterministic: true, conjunction: 1, output_deterministic: ['foo'])
+  Nokogiri::XML::Builder.new do |xml|
+    xml.project(task: task) {
+      xml.send('data-sources') {
+        ids.each_with_index do |id,i|
+          xml.send('data-source', id: id, file: files[i], type: types[i], :'field-separator' => field_separators[i]) {
+            xml.fields {
+              next unless fields[i]
+              fields[i].each do |field|
+                xml.field(field)
+              end
+            }
+          }
+        end
+      }
+      if deterministic
+        xml.send('deterministic-linkage') {
+          conjunction.times do
+            xml.conjunction {
+              parts.each do |part|
+                xml.part(part)
+              end
+            }
+          end
+        }
+      end
+      if output_deterministic
+        if output_deterministic.empty?
+          xml.output
+        else
+          output_deterministic.each do |out|
+            xml.output(deterministic: out)
+          end
+        end
+      end
+    }
+  end.to_xml
+end
+
+RSpec.describe Parser do
+  context "initialization" do
+    it "expects a xml string as argument" do
+      expect {Parser.new}.to raise_error(ArgumentError)
+    end
+    it "complains about invalid XML" do
+      bad_xml = '<?xml version="1.0 encoding="UTF-8"?>'
+      expect {Parser.new(bad_xml)}.to raise_error(Parser::SyntaxError)
+    end
+  end
+
+  context "tasks" do
+    it "accepts only valid tasks" do
+      xml = xml_dedup
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_linkage
+      expect {Parser.new(xml)}.to_not raise_error
+    end
+
+    it "rejects invalid tasks" do
+      xml = xml_dedup(task: 'foo')
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidTaskError, "Invalid task: 'foo'")
+    end
+  end
+
+  context "data source" do
+    it "accepts only one data sources when deduplicating" do
+      xml = xml_linkage(task: 'deduplication')
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidNumberOfSources, "Deduplication: expected one data-sources, 2 given")
+    end
+
+    it "accepts only two data sources when linking" do
+      xml = xml_dedup(task: 'linkage')
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidNumberOfSources, "Linkage: expected two data-source, 1 given")
+    end
+
+    it "rejects data sources without id" do
+      xml = xml_dedup(ids: [nil])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingDataSourceId)
+    end
+
+    it "rejects two data sources with same id" do
+      xml = xml_linkage(ids: [1,1])
+      expect {Parser.new(xml)}.to raise_error(Parser::DuplicatedDataSourceId, "Duplicated data source id '1'")
+    end
+
+    it "expects a file name" do
+      xml = xml_dedup(files: [''])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingDataSourceFile, "Missing file attribute from data source '0'")
+    end
+
+    it "rejects invalid file names" do
+      xml = xml_dedup(files: ['invalid'])
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidDataSourceFile, "File 'invalid' from data source '0' does not exist")
+    end
+
+    it "accepts only 'delimiter' type" do
+      xml = xml_dedup(types: ['delimited'])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(types: ['foo'])
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidDataSourceType, "Data source type 'foo' not supported")
+    end
+
+    it "expects a field-separator" do
+      xml = xml_dedup(field_separators: ["\t"])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(field_separators: [""])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingFieldSeparator, "Missing field separator from data source '0'")
+    end
+
+    it "expects a 'fields' entry" do
+      xml = xml_dedup(fields: [[{name: 'id', type: 'int'}]], parts: [{:'field-name' => 'id'}])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(fields: [])
+      expect {Parser.new(xml)}.to raise_error(Parser::FieldsElementNotFound)
+    end
+
+    it "expect a name attribute on field element" do
+      xml = xml_dedup(fields: [[{name: '', type: 'int'}]])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingFieldName)
+    end
+
+    it "rejects duplicated names on the same data source" do
+      xml = xml_dedup(fields: [[{name: 'foo', type: 'int'},{name: 'foo', type: 'int'}]])
+      expect {Parser.new(xml)}.to raise_error(Parser::DuplicatedFieldName, "Duplicated field name 'foo' on data source '0'")
+    end
+
+    it "expect a valid field type" do
+      ['int', 'string'].each do |type|
+        xml = xml_dedup(fields: [[{name: 'foo', type: type}]], parts: [{:'field-name' => 'foo'}])
+        expect {Parser.new(xml)}.to_not raise_error
+      end
+
+      xml = xml_dedup(fields: [[{name: 'foo', type: 'bar'}]])
+      expect {Parser.new(xml)}.to raise_error(Parser::InvalidFieldType, "Invalid type 'bar' from field 'foo', data source '0'")
+    end
+  end
+
+  context "deterministic" do
+    it "expects a deterministic-linkage element" do
+      xml = xml_dedup(deterministic: false)
+      expect {Parser.new(xml)}.to raise_error(Parser::DeterministicLinkageElementNotFound, "Missing deterministic-linkage element")
+    end
+
+    it "expects a single conjunction element" do
+      xml = xml_dedup(conjunction: 0)
+      expect {Parser.new(xml)}.to raise_error(Parser::ConjunctionElementNotFound, "Missing conjunction element")
+
+      xml = xml_dedup(conjunction: 1)
+      expect {Parser.new(xml)}.to_not raise_error
+
+      conjunction = 2
+      xml = xml_dedup(conjunction: conjunction)
+      expect {Parser.new(xml)}.to raise_error(Parser::MultipleConjunctionElements, "Only one conjunction element is allowed, #{conjunction} found")
+    end
+
+    it "expects at leas one part element" do
+      xml = xml_dedup(fields: [[{name: 'foo', type: 'int'}]], parts: [{:'field-name' => 'foo'}])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(parts: [])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingPart, "At leas one part element is required")
+    end
+
+    it "expects a field-name on part elements" do
+      xml = xml_dedup(fields: [[{name: 'foo', type: 'string'}]], parts: [{:'field-name' => 'foo'}])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(parts: [{not_field_name: 'foo'}])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingFieldName, "Missing attribute field-name on part element")
+    end
+
+    it "accepts only field-names present on all data sources" do
+      xml = xml_linkage(fields: [[{name: 'foo', type: 'string'}],[{name: 'foo', type: 'string'}]], parts: [{:'field-name' => 'foo'}])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_linkage(fields: [[{name: 'foo', type: 'string'}],[{name: 'bar', type: 'string'}]], parts: [{:'field-name' => 'foo'}])
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingPartFieldNameOnDataSource, "Field name 'foo' not found on data source '1'")
+    end
+  end
+
+  context "output" do
+    it "expects a single 'output' element" do
+      xml = xml_dedup(output_deterministic: ["foo"])
+      expect {Parser.new(xml)}.to_not raise_error
+
+      xml = xml_dedup(output_deterministic: ["foo","bar"])
+      expect {Parser.new(xml)}.to raise_error(Parser::MultipleOutputElement, "Only one output element is allowed, 2 found")
+
+      xml = xml_dedup(output_deterministic: false)
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingOutputElement, "Missing output element")
+    end
+
+    it "expects a deterministic attribute on output element" do
+      xml = xml_dedup(output_deterministic: "")
+      expect {Parser.new(xml)}.to raise_error(Parser::MissingDeterministicAttribute, "Missing attribute 'deterministic' on output element")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rpareia'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.disable_monkey_patching!
+
+  #config.warnings = true
+
+  if config.files_to_run.one?
+    config.default_formatter = 'doc'
+  end
+
+  #config.profile_examples = 10
+
+  config.order = :random
+
+  Kernel.srand config.seed
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: rpareia
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Michel Boaventura
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rbczmq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.12'
+description: A data deduplication software, based on Pareia
+email:
+- michel.boaventura@gmail.com
+executables:
+- rpareia
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/rpareia
+- lib/rpareia.rb
+- lib/rpareia/blocker.rb
+- lib/rpareia/heartbeat.rb
+- lib/rpareia/parser.rb
+- lib/rpareia/reader.rb
+- lib/rpareia/version.rb
+- rpareia.gemspec
+- spec/fixtures/empty_file.xml
+- spec/rpareia/parser_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/michelboaventura/rpareia
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Pareia implementation, with ruby and ZeroMQ
+test_files:
+- spec/fixtures/empty_file.xml
+- spec/rpareia/parser_spec.rb
+- spec/spec_helper.rb