corpus-processor 0.2.0 → 0.3.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: corpus-processor
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Das Dad
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-01 00:00:00.000000000 Z
+date: 2013-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -42,18 +56,18 @@ dependencies:
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.14.0.rc1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.14.0.rc1
 - !ruby/object:Gem::Dependency
-  name: fakefs
+  name: pry-nav
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -67,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-nav
+  name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -82,7 +96,7 @@ dependencies:
         version: '0'
 description: Process linguistic corpus
 email:
-- feedback@dasdad.com.br
+- dev@dasdad.com.br
 executables:
 - corpus-processor
 extensions: []
@@ -90,30 +104,29 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .rspec
+- .travis.yml
+- .yardopts
 - Gemfile
 - README.md
 - bin/corpus-processor
 - corpus-processor.gemspec
 - lib/corpus-processor.rb
+- lib/corpus-processor/categories.rb
+- lib/corpus-processor/categories/default.yml
 - lib/corpus-processor/cli.rb
-- lib/corpus-processor/default_categories.rb
 - lib/corpus-processor/generators.rb
 - lib/corpus-processor/generators/stanford_ner.rb
 - lib/corpus-processor/parsers.rb
 - lib/corpus-processor/parsers/lampada.rb
 - lib/corpus-processor/processor.rb
 - lib/corpus-processor/token.rb
-- lib/corpus-processor/tokenizer.rb
-- lib/corpus-processor/traverser.rb
 - lib/corpus-processor/version.rb
-- spec/integration/cli_spec.rb
+- spec/corpus-processor/cli_spec.rb
+- spec/corpus-processor/generators/stanford_ner_spec.rb
+- spec/corpus-processor/parsers/lampada_spec.rb
+- spec/corpus-processor/processor_spec.rb
+- spec/corpus-processor/token_spec.rb
 - spec/spec_helper.rb
-- spec/unit/generators/stanford_ner_spec.rb
-- spec/unit/parsers/lampada_spec.rb
-- spec/unit/processor.rb
-- spec/unit/token_spec.rb
-- spec/unit/tokenizer_spec.rb
-- spec/unit/traverser_spec.rb
 homepage: https://github.com/dasdad/corpus-processor
 licenses:
 - MIT
@@ -134,16 +147,15 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.0.2
 signing_key: 
 specification_version: 4
 summary: Handle linguistic corpus and convert it to use NLP tools
 test_files:
-- spec/integration/cli_spec.rb
+- spec/corpus-processor/cli_spec.rb
+- spec/corpus-processor/generators/stanford_ner_spec.rb
+- spec/corpus-processor/parsers/lampada_spec.rb
+- spec/corpus-processor/processor_spec.rb
+- spec/corpus-processor/token_spec.rb
 - spec/spec_helper.rb
-- spec/unit/generators/stanford_ner_spec.rb
-- spec/unit/parsers/lampada_spec.rb
-- spec/unit/processor.rb
-- spec/unit/token_spec.rb
-- spec/unit/tokenizer_spec.rb
-- spec/unit/traverser_spec.rb
+has_rdoc: 

data/lib/corpus-processor/default_categories.rb

@@ -1,14 +0,0 @@
-module CorpusProcessor
-  DEFAULT_CATEGORIES = {
-    input: {
-      "PESSOA"      => :person,
-      "LOCAL"       => :location,
-      "ORGANIZACAO" => :organization,
-    },
-    output: Hash.new("O").merge(
-        person:       "PERSON",
-        location:     "LOCATION",
-        organization: "ORGANIZATION",
-    )
-  }
-end

data/lib/corpus-processor/tokenizer.rb

@@ -1,17 +0,0 @@
-class CorpusProcessor::Tokenizer
-  def tokenize(text, category = nil)
-    strip_tags(text)
-      .gsub(/[[:punct:]]/, "")
-      .strip
-      .split(/\s+/)
-      .map { |word| CorpusProcessor::Token.new(word, category) }
-  end
-
-  def strip_tags(text)
-    text.gsub(/<.*?>/, " ").strip
-  end
-
-  def join_lines(text)
-    text.gsub(/\s+/, " ").strip
-  end
-end

data/lib/corpus-processor/traverser.rb

@@ -1,19 +0,0 @@
-class CorpusProcessor::Traverser
-  def traverse(text, regexp, &block)
-    return if block.nil?
-    remaining_search = text
-    until remaining_search.empty?
-      match = remaining_search.match(regexp)
-      if match.nil?
-        block.call remaining_search unless remaining_search.empty?
-        remaining_search = ""
-      else
-        before           = remaining_search[0...match.begin(0)]
-        remaining_search = remaining_search[match.end(0)..-1]
-
-        block.call before unless before.empty?
-        block.call match
-      end
-    end
-  end
-end

data/spec/unit/generators/stanford_ner_spec.rb

@@ -1,46 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Generators::StanfordNer do
-  subject(:stanford_ner) { CorpusProcessor::Generators::StanfordNer.new }
-
-  describe "#generate" do
-    subject { stanford_ner.generate(tokens) }
-
-    context "no tokens" do
-      let(:tokens) { [] }
-
-      it { should == "\n" }
-    end
-
-    context "one token" do
-      let(:tokens) { [CorpusProcessor::Token.new("banana")] }
-
-      it { should == "banana	O\n" }
-    end
-
-    context "two tokens" do
-      let(:tokens) { [
-        CorpusProcessor::Token.new("good"),
-        CorpusProcessor::Token.new("banana"),
-      ] }
-
-      it { should == "good	O\nbanana	O\n" }
-    end
-
-    context "with category" do
-      let(:tokens) { [CorpusProcessor::Token.new("Leandro", :person)] }
-
-      it { should == "Leandro	PERSON\n" }
-    end
-
-    context "with non-default categories" do
-      let(:stanford_ner) { CorpusProcessor::Generators::StanfordNer.new(
-        banana: "BANANA"
-      ) }
-
-      let(:tokens) { [CorpusProcessor::Token.new("Nanica", :banana)] }
-
-      it { should == "Nanica	BANANA\n" }
-    end
-  end
-end

data/spec/unit/parsers/lampada_spec.rb

@@ -1,269 +0,0 @@
-require "spec_helper"
-
-describe CorpusProcessor::Parsers::Lampada do
-  subject(:lampada) { CorpusProcessor::Parsers::Lampada.new }
-
-  describe "#parse" do
-    subject { lampada.parse(corpus) }
-
-    context "default categories" do
-      context "empty corpus" do
-        let(:corpus) { "" }
-
-        it { should == [] }
-      end
-
-      context "doctype" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-CORPUS
-        }
-
-        it { should == [] }
-      end
-
-      context "simple phrase" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-<DOC DOCID="H2-dftre765">
-  <P>Fatores Demográficos e Econômicos Subjacentes</P>
-</DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("Fatores"),
-            CorpusProcessor::Token.new("Demográficos"),
-            CorpusProcessor::Token.new("e"),
-            CorpusProcessor::Token.new("Econômicos"),
-            CorpusProcessor::Token.new("Subjacentes"),
-          ]
-        }
-      end
-
-      context "two simple phrases" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-<DOC DOCID="H2-dftre765">
-  <P>Fatores Demográficos e Econômicos Subjacentes</P>
-  <P>Fatores Demográficos e Econômicos Subjacentes</P>
-</DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("Fatores"),
-            CorpusProcessor::Token.new("Demográficos"),
-            CorpusProcessor::Token.new("e"),
-            CorpusProcessor::Token.new("Econômicos"),
-            CorpusProcessor::Token.new("Subjacentes"),
-            CorpusProcessor::Token.new("Fatores"),
-            CorpusProcessor::Token.new("Demográficos"),
-            CorpusProcessor::Token.new("e"),
-            CorpusProcessor::Token.new("Econômicos"),
-            CorpusProcessor::Token.new("Subjacentes"),
-          ]
-        }
-      end
-
-      context "useless entity" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-<DOC DOCID="H2-dftre765">
-  <P>Nos finais da <EM ID="H2-dftre765-102" CATEG="OUTRO" COMENT="DUVIDA_DIRECTIVASTEMPO">Idade Média</EM></P>
-</DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("Nos"),
-            CorpusProcessor::Token.new("finais"),
-            CorpusProcessor::Token.new("da"),
-            CorpusProcessor::Token.new("Idade"),
-            CorpusProcessor::Token.new("Média"),
-          ]
-        }
-      end
-
-      context "one entity" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-  <DOC DOCID="H2-dftre765">
-      <P>Foram igualmente determinantes para evitar que as ideias reformadoras encontrassem divulgação em
-      <EM ID="H2-dftre765-23" CATEG="LOCAL" TIPO="HUMANO" SUBTIPO="PAIS" COREL="H2-dftre765-8 H2-dftre765-37" TIPOREL="local_nascimento_de incluido">Portugal</EM>
-      </P>
-  </DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("Foram"),
-            CorpusProcessor::Token.new("igualmente"),
-            CorpusProcessor::Token.new("determinantes"),
-            CorpusProcessor::Token.new("para"),
-            CorpusProcessor::Token.new("evitar"),
-            CorpusProcessor::Token.new("que"),
-            CorpusProcessor::Token.new("as"),
-            CorpusProcessor::Token.new("ideias"),
-            CorpusProcessor::Token.new("reformadoras"),
-            CorpusProcessor::Token.new("encontrassem"),
-            CorpusProcessor::Token.new("divulgação"),
-            CorpusProcessor::Token.new("em"),
-            CorpusProcessor::Token.new("Portugal", :location),
-          ]
-        }
-      end
-
-      context "multiple entities" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-  <DOC DOCID="H2-dftre765">
-      <P>
-      A imprensa, inventada na
-      <EM ID="H2-dftre765-9" CATEG="LOCAL" TIPO="HUMANO" SUBTIPO="PAIS" COREL="H2-dftre765-37" TIPOREL="incluido">Alemanha</EM>
-      por
-      <EM ID="H2-dftre765-10" CATEG="PESSOA" TIPO="INDIVIDUAL" COREL="H2-dftre765-9" TIPOREL="natural_de">John Gutenberg</EM>
-      <EM ID="H2-dftre765-20" CATEG="ORGANIZACAO" TIPO="INSTITUICAO" COMENT="2/3">Inquisição</EM>
-      e a censura 
-      </P>
-  </DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("A"),
-            CorpusProcessor::Token.new("imprensa"),
-            CorpusProcessor::Token.new("inventada"),
-            CorpusProcessor::Token.new("na"),
-            CorpusProcessor::Token.new("Alemanha", :location),
-            CorpusProcessor::Token.new("por"),
-            CorpusProcessor::Token.new("John", :person),
-            CorpusProcessor::Token.new("Gutenberg", :person),
-            CorpusProcessor::Token.new("Inquisição", :organization),
-            CorpusProcessor::Token.new("e"),
-            CorpusProcessor::Token.new("a"),
-            CorpusProcessor::Token.new("censura"),
-          ]
-        }
-      end
-
-      context "spaces after ponctuation" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-  <DOC DOCID="H2-dftre765">
-      <EM ID="H2-dftre765-1" CATEG="ABSTRACCAO|ACONTECIMENTO" TIPO="IDEIA|EFEMERIDE">Reforma Protestante</EM>
-      . No
-  </DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("Reforma"),
-            CorpusProcessor::Token.new("Protestante"),
-            CorpusProcessor::Token.new("No"),
-          ]
-        }
-      end
-    end
-
-    context "user-defined categories" do
-      let(:lampada) {
-        CorpusProcessor::Parsers::Lampada.new({
-          "FRUTA" => :fruit,
-          "LIVRO" => :book,
-        })
-      }
-
-      context "multiple entities" do
-        let(:corpus) {
-<<-CORPUS
-<?xml version="1.0" encoding="ISO-8859-1"?>
-<!DOCTYPE colHAREM>
-<colHAREM versao="Segundo_dourada_com_relacoes_14Abril2010">
-  <DOC DOCID="H2-dftre765">
-      <P>
-      A imprensa, inventada na
-      <EM ID="H2-dftre765-9" CATEG="FRUTA" TIPO="HUMANO" SUBTIPO="PAIS" COREL="H2-dftre765-37" TIPOREL="incluido">Banana</EM>
-      por
-      <EM ID="H2-dftre765-10" CATEG="LIVRO" TIPO="INDIVIDUAL" COREL="H2-dftre765-9" TIPOREL="natural_de">Harry Potter</EM>
-      <EM ID="H2-dftre765-20" CATEG="ORGANIZACAO" TIPO="INSTITUICAO" COMENT="2/3">Inquisição</EM>
-      e a censura 
-      </P>
-  </DOC>
-</colHAREM>
-CORPUS
-        }
-
-        it { should == [
-            CorpusProcessor::Token.new("A"),
-            CorpusProcessor::Token.new("imprensa"),
-            CorpusProcessor::Token.new("inventada"),
-            CorpusProcessor::Token.new("na"),
-            CorpusProcessor::Token.new("Banana", :fruit),
-            CorpusProcessor::Token.new("por"),
-            CorpusProcessor::Token.new("Harry", :book),
-            CorpusProcessor::Token.new("Potter", :book),
-            CorpusProcessor::Token.new("Inquisição"),
-            CorpusProcessor::Token.new("e"),
-            CorpusProcessor::Token.new("a"),
-            CorpusProcessor::Token.new("censura"),
-          ]
-        }
-      end
-    end
-  end
-
-  describe "#extract_category" do
-    subject { lampada.extract_category(categories) }
-
-    context "empty categories" do
-      let(:categories) { "" }
-
-      it { should == nil }
-    end
-
-    context "one category" do
-      let(:categories) { "PESSOA" }
-
-      it { should == :person }
-    end
-
-    context "two categories" do
-      let(:categories) { "OUTRA|ORGANIZACAO" }
-
-      it { should == :organization }
-    end
-
-    context "ambiguidade" do
-      let(:categories) { "PESSOA|ORGANIZACAO" }
-
-      it { should == :person }
-    end
-  end
-end