RubyGems - vn_tagger - Versions diffs - 0.0.4 - Mend

vn_tagger 0.0.4

Files changed (31) hide show

checksums.yaml +7 -0
data/.gitignore +28 -0
data/Gemfile +5 -0
data/LICENSE.txt +22 -0
data/README.md +34 -0
data/Rakefile +2 -0
data/VN_TAGGER LICENSE.txt +5 -0
data/lib/commons-cli-1.2.jar +0 -0
data/lib/edu.stanford.nlp.tagger-2.0.jar +0 -0
data/lib/vn.hus.nlp.fsm-1.0.0.jar +0 -0
data/lib/vn.hus.nlp.tokenizer-4.1.1.jar +0 -0
data/lib/vn.hus.nlp.utils-1.0.0.jar +0 -0
data/lib/vn_tagger.rb +7 -0
data/lib/vn_tagger/tagger.rb +44 -0
data/lib/vn_tagger/version.rb +3 -0
data/resources/automata/dfaLexicon.xml +123032 -0
data/resources/automata/externalLexicon.xml +7 -0
data/resources/bigram/bigram.xml +211658 -0
data/resources/bigram/unigram.xml +24309 -0
data/resources/lexers/lexers.xml +68 -0
data/resources/models/vtb.tagger +0 -0
data/resources/models/vtb.tagger.props +36 -0
data/resources/normalization/rules.txt +15 -0
data/resources/prefix/namedEntityPrefix.xml +34 -0
data/spec/spec_helper.rb +9 -0
data/spec/vn_tagger/tagger_spec.rb +14 -0
data/spec/vn_tagger_spec.rb +6 -0
data/vn.hus.nlp.tagger-4.2.0.jar +0 -0
data/vnTagger.sh +28 -0
data/vn_tagger.gemspec +25 -0
metadata +134 -0

data/resources/lexers/lexers.xml ADDED Viewed

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="utf-8" standalone="yes"?>
+<!--  (C) phuonglh@gmail.com -->
+<corpus id="resources/lexers/lexers.xml">
+    <body>
+        <w msd="numbersign">#</w>
+        <w msd="ampersand">&amp;</w>
+        <w msd="date_mm-dd-yy">(0*[1-9]|1[012])-(0*[1-9]|[12][0-9]|3[01])-\d\d</w>
+        <w msd="date_mm/dd/yy">(0*[1-9]|1[012])/(0*[1-9]|[12][0-9]|3[01])/\d\d</w>
+        <w msd="date_mm.dd.yy">(0*[1-9]|1[012])\.(0*[1-9]|[12][0-9]|3[01])\.\d\d</w>
+        <w msd="date_dd-mm-yy">([12][0-9]|3[01]|0*[1-9])-(1[012]||0*[1-9])-\d\d</w>
+        <w msd="date_dd/mm/yy">([12][0-9]|3[01]|0*[1-9])/(1[012]||0*[1-9])/\d\d</w>
+        <w msd="date_dd.mm.yy">([12][0-9]|3[01]|0*[1-9])[\.](1[012]||0*[1-9])[\.]\d\d</w>
+        <w msd="date_dd-mm-yyyy">([12][0-9]|3[01]|0*[1-9])-(1[012]||0*[1-9])-(19|20)\d\d</w>
+        <w msd="date_dd/mm/yyyy">([12][0-9]|3[01]|0*[1-9])/(1[012]||0*[1-9])/(19|20)\d\d</w>
+        <w msd="date_dd.mm.yyyy">([12][0-9]|3[01]|0*[1-9])\.(1[012]||0*[1-9])\.(19|20)\d\d</w>
+        <w msd="date_dd-mm">(0*[1-9]|[12][0-9]|3[01])[-/\.](1[012]|0*[1-9])</w>
+        <w msd="date_mm-yy">(0*[1-9]|1[012])[-/\.]\d\d</w>
+        <w msd="date_mm-yyyy">(0*[1-9]|1[012])[-/\.](19|20)\d\d</w>
+        <w msd="date_yyyy">(19|20)\d\d</w>
+        <w msd="date_mm-dd-yyyy">(0*[1-9]|1[012])-([12][0-9]|3[01]|0*[1-9])-(19|20)\d\d</w>
+        <w msd="date_mm/dd/yyyy">(0*[1-9]|1[012])/([12][0-9]|3[01]|0*[1-9])/(19|20)\d\d</w>
+        <w msd="date_mm.dd.yyyy">(0*[1-9]|1[012])\.([12][0-9]|3[01]|0*[1-9])\.(19|20)\d\d</w>
+        <w msd="date_yyyy-mm-dd">(19|20)\d\d-(0*[1-9]|1[012])-([12][0-9]|3[01]|0*[1-9])</w>
+        <w msd="date_yyyy/mm/dd">(19|20)\d\d/(0*[1-9]|1[012])/([12][0-9]|3[01]|0*[1-9])</w>
+        <w msd="date_yyyy.mm.dd">(19|20)\d\d\.(0*[1-9]|1[012])\.([12][0-9]|3[01]|0*[1-9])</w>
+        <w msd="hhmmss">([0-1]\d|[2][0-3]):[0-5]\d:[0-5]\d</w>
+        <w msd="percent">([0-9]*[\.,])?[0-9]+%</w>
+        <w msd="name1">[A-ZÁÂĐÍÔƯỨÝỶ][aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]*</w>
+        <w msd="name2">([A-ZÁÂĐÍÔƯỨÝỶ][aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]*)(\s+[A-ZÁÂĐÍÔƯỨÝỶ][aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]+)*</w>
+        <w msd="phrase">([AÀẢÃÁẠĂẰẲẴẮẶÂẦẨẪẤẬBCDĐEÈẺẼÉẸÊỀỂỄẾỆFGHIÌỈĨÍỊJKLMNOÒỎÕÓỌÔỒỔỖỐỘƠỜỞỠỚỢPQRSTUÙỦŨÚỤƯỪỬỮỨỰVWXYỲỶỸÝỴZ])?([aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz\s])*([aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz])+$*</w>
+        <w msd="allcaps">([AÀẢÃÁẠĂẰẲẴẮẶÂẦẨẪẤẬBCDĐEÈẺẼÉẸÊỀỂỄẾỆFGHIÌỈĨÍỊJKLMNOÒỎÕÓỌÔỒỔỖỐỘƠỜỞỠỚỢPQRSTUÙỦŨÚỤƯỪỬỮỨỰVWXYỲỶỸÝỴZ]+)(\s*[AÀẢÃÁẠĂẰẲẴẮẶÂẦẨẪẤẬBCDĐEÈẺẼÉẸÊỀỂỄẾỆFGHIÌỈĨÍỊJKLMNOÒỎÕÓỌÔỒỔỖỐỘƠỜỞỠỚỢPQRSTUÙỦŨÚỤƯỪỬỮỨỰVWXYỲỶỸÝỴZ]+)*[^aàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz/\)\(\?!\.;:,\-"']</w>
+        <w msd="fraction">(\d+)/(\d+)</w>
+        <w msd="email">(\w[-._\w]*\w@\w[-._\w]*\w\.\w{2,3})</w>
+        <w msd="return">(^$)</w>
+        <w msd="fslash">/</w>
+        <w msd="langle">&lt;</w>
+        <w msd="xmltags">&lt;/*\w*&gt;</w>
+        <w msd="equal">=</w>
+        <w msd="rangle">&gt;</w>
+        <w msd="aroba">@</w>
+        <w msd="number1">[+]?([0-9]*)?[0-9]+([\.,]\d+)*</w>
+        <!--
+        <w msd="number2">[+]?([0-9]*)?[0-9]+([\.,]\d+)*(\s|tỉ|tỷ|triệu|ngàn|nghìn|trăm|chục)*</w>
+        -->
+        <w msd="degree">[-+]?([0-9]*[\.,])?[0-9]+°</w>
+        <w msd="ponctuation">[\\?!\\.:;,\-"']</w>
+        <w msd="dollar">\$</w>
+        <w msd="lparen">\(</w>
+        <w msd="rparen">\)</w>
+        <w msd="asterisk">\*</w>
+        <w msd="plus">\+</w>
+        <w msd="minus">\-</w>
+        <w msd="ellipsis">\.\.\.</w>
+        <w msd="residual">\W</w>
+        <w msd="lbracket">\[</w>
+        <w msd="bslash">\\</w>
+        <w msd="rbracket">\]</w>
+        <w msd="entity0">\d+([\.,]\d+)*[A-Zaàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]+\d+$</w>
+        <w msd="entity1">[A-Zaàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]+(\d)*$</w>
+        <w msd="entity2">[A-Zaàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz\d]+([\.\-/][\dA-Zaàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]+)*[\dA-Zaàảãáạăằẳẵắặâầẩẫấậbcdđeèẻẽéẹêềểễếệfghiìỉĩíịjklmnoòỏõóọôồổỗốộơờởỡớợpqrstuùủũúụưừửữứựvwxyỳỷỹýỵz]+</w>
+        <w msd="space">\s+</w>
+        <w msd="word">\w</w>
+        <w msd="lcbrace">\{</w>
+        <w msd="rcbrace">\}</w>
+        <w msd="underscore">_</w>
+        <w msd="pound">£</w>
+    </body>
+</corpus>

data/resources/models/vtb.tagger ADDED Viewed

Binary file

data/resources/models/vtb.tagger.props ADDED Viewed

@@ -0,0 +1,36 @@
+## tagger training invoked at Thu Aug 05 14:33:50 CEST 2010 with arguments:
+                   model = resources/models/vtb.tagger
+                    arch = left5words,vietnameseunknowns
+               trainFile = data/vtb-20091030.tagged.txt
+         closedClassTags =
+ closedClassTagThreshold = 40
+ curWordMinFeatureThresh = 2
+                   debug = true
+             debugPrefix =
+            tagSeparator = /
+                encoding = UTF-8
+           initFromTrees = false
+              iterations = 100
+                    lang = vietnamese
+    learnClosedClassTags = false
+        minFeatureThresh = 2
+           openClassTags =
+rareWordMinFeatureThresh = 10
+          rareWordThresh = 6
+                  search = qn
+                    sgml = false
+            sigmaSquared = 0.5
+                   regL1 = 1.0
+               tagInside =
+                tokenize = false
+        tokenizerFactory =
+        tokenizerOptions =
+               treeRange =
+          treeNormalizer =
+         treeTransformer =
+                 verbose = false
+    veryCommonWordThresh = 50
+                xmlInput =
+              outputFile =
+            outputFormat = slashTags
+     outputFormatOptions =

data/resources/normalization/rules.txt ADDED Viewed

@@ -0,0 +1,15 @@
+òa	oà
+óa	oá
+ỏa	oả
+õa	oã
+ọa	oạ
+òe	oè
+óe	oé
+ỏe	oẻ
+õe	oẽ
+ọe	oẹ
+ùy	uỳ
+úy	uý
+ủy	uỷ
+ũy	uỹ
+ụy	uỵ

data/resources/prefix/namedEntityPrefix.xml ADDED Viewed

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8" standalone="yes"?>
+<corpus id="resources/prefix/namedEntityPrefix.xml">
+    <body>
+		<!-- classificator prefix -->
+		<w>ông</w>
+		<w>bà</w>
+		<w>bác</w>
+		<w>chú</w>
+		<w>cô</w>
+		<w>thím</w>
+		<w>dì</w>
+		<w>cậu</w>
+		<w>cụ</w>
+		<w>mợ</w>
+		<w>ngài</w>
+		<w>anh</w>
+		<w>chị</w>
+		<w>thằng</w>
+		<w>cái</w>
+		<w>vua</w>
+		<!-- organization prefix -->
+		<w>tỉnh</w>
+		<w>huyện</w>
+		<w>xã</w>
+		<w>phường</w>
+		<w>bộ</w>
+		<w>sở</w>
+		<w>cục</w>
+		<w>ban</w>
+		<w>ngành</w>
+		<!-- Other prefix -->
+		<w>theo</w>
+    </body>
+</corpus>

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,9 @@
+require 'byebug'
+spec_folder = File.dirname(__FILE__)
+root_folder = File.dirname(spec_folder)
+app_folder = File.join(root_folder, 'lib', '*.rb')
+Dir[app_folder].each { |file| require file }
+RSpec.configure do |config|
+end

data/spec/vn_tagger/tagger_spec.rb ADDED Viewed

@@ -0,0 +1,14 @@
+require 'spec_helper'
+require 'byebug'
+describe VnTagger::Tagger do
+  describe '#tag' do
+    let(:text) { 'HLV cùa Chelsea không hối tiếc vì hành động bỏ về sớm trong trận gặp Aston Villa.' }
+    let(:tagger) { described_class.new(text) }
+    let(:result) { tagger.tag }
+    it 'returns xml tagged text' do
+      expect(result).to be_a(Nokogiri::XML::Document)
+    end
+  end
+end

data/spec/vn_tagger_spec.rb ADDED Viewed

@@ -0,0 +1,6 @@
+require 'spec_helper'
+describe VnTagger do
+  describe '#tokenize' do
+  end
+end

data/vn.hus.nlp.tagger-4.2.0.jar ADDED Viewed

Binary file

data/vnTagger.sh ADDED Viewed

@@ -0,0 +1,28 @@
+#!/bin/sh
+# The main program to run
+PROGRAM="${BASH_SOURCE%/*}/vn.hus.nlp.tagger-4.2.0.jar"
+# Get the java command
+#
+if [ -z "$JAVACMD" ] ; then
+  if [ -n "$JAVA_HOME"  ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+    fi
+  else
+    JAVACMD=`which java 2> /dev/null`
+    if [ -z "$JAVACMD" ] ; then
+      JAVACMD=java
+    fi
+  fi
+fi
+# Run the programme
+#
+$JAVACMD -mx500m  -jar $PROGRAM $@

data/vn_tagger.gemspec ADDED Viewed

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'vn_tagger/version'
+Gem::Specification.new do |spec|
+  spec.name          = "vn_tagger"
+  spec.version       = VnTagger::VERSION
+  spec.authors       = ["Hieu Nguyen"]
+  spec.email         = ["hieuk09@gmail.com"]
+  spec.summary       = %q{This is a wrapper for vn_tagger library, a A POS tagger for Vietnamese texts.' }
+  spec.description   = %q{This is a wrapper for vn_tagger library, a A POS tagger for Vietnamese texts.' }
+  spec.homepage      = ""
+  spec.license       = "MIT"
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib", 'lib/vn_tagger']
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "byebug"
+end

metadata ADDED Viewed

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: vn_tagger
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Hieu Nguyen
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-12-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'This is a wrapper for vn_tagger library, a A POS tagger for Vietnamese
+  texts.'' '
+email:
+- hieuk09@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- VN_TAGGER LICENSE.txt
+- lib/commons-cli-1.2.jar
+- lib/edu.stanford.nlp.tagger-2.0.jar
+- lib/vn.hus.nlp.fsm-1.0.0.jar
+- lib/vn.hus.nlp.tokenizer-4.1.1.jar
+- lib/vn.hus.nlp.utils-1.0.0.jar
+- lib/vn_tagger.rb
+- lib/vn_tagger/tagger.rb
+- lib/vn_tagger/version.rb
+- resources/automata/dfaLexicon.xml
+- resources/automata/externalLexicon.xml
+- resources/bigram/bigram.xml
+- resources/bigram/unigram.xml
+- resources/lexers/lexers.xml
+- resources/models/vtb.tagger
+- resources/models/vtb.tagger.props
+- resources/normalization/rules.txt
+- resources/prefix/namedEntityPrefix.xml
+- spec/spec_helper.rb
+- spec/vn_tagger/tagger_spec.rb
+- spec/vn_tagger_spec.rb
+- vn.hus.nlp.tagger-4.2.0.jar
+- vnTagger.sh
+- vn_tagger.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- lib/vn_tagger
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.3
+signing_key:
+specification_version: 4
+summary: This is a wrapper for vn_tagger library, a A POS tagger for Vietnamese texts.'
+test_files:
+- spec/spec_helper.rb
+- spec/vn_tagger/tagger_spec.rb
+- spec/vn_tagger_spec.rb