opener-language-identifier 3.0.0

data/lib/opener/language_identifier/cli.rb

@@ -0,0 +1,139 @@
+module Opener
+  class LanguageIdentifier
+    ##
+    # CLI wrapper around {Opener::LanguageIdentifier} using OptionParser.
+    #
+    # @!attribute [r] options
+    #  @return [Hash]
+    #
+    # @!attribute [r] option_parser
+    #  @return [OptionParser]
+    #
+    class CLI
+      attr_reader :options, :option_parser
+
+      ##
+      # @param [Hash] options
+      #
+      def initialize(options = {})
+        @options = DEFAULT_OPTIONS.merge(options)
+
+        @option_parser = OptionParser.new do |opts|
+          opts.program_name   = 'language-identifier'
+          opts.summary_indent = '  '
+
+          opts.on('-v', '--version', 'Shows the current version') do
+            show_version
+          end
+
+          opts.on('-k', '--[no-]kaf', 'Output the language as KAF') do |v|
+            @options[:kaf] = v
+          end
+
+          opts.on('-p', '--probs', 'Provide probabilities, assumes --no-kaf') do
+            @options[:kaf] = false
+            @options[:probs] = true
+          end
+
+          opts.separator <<-EOF
+
+Examples:
+
+  cat example_text.txt | #{opts.program_name}    # Basic detection
+
+Languages:
+
+  * ar	Arabic
+  * bg	Bulgarian
+  * bn	Bengali
+  * cs	Czech
+  * da	Danish
+  * de	German
+  * el	Greek
+  * en	English
+  * es	Spanish
+  * et	Estonian
+  * fa	Persian
+  * fi	Finnish
+  * fr	French
+  * gu	Gujarati
+  * he	Hebrew
+  * hi	Hindi
+  * hr	Croatian
+  * hu	Hungarian
+  * id	Indonesian
+  * it	Italian
+  * ja	Japanese
+  * kn	Kannada
+  * ko	Korean
+  * lt	Lithuanian
+  * lv	Latvian
+  * mk	Macedonian
+  * ml	Malayalam
+  * mr	Marathi
+  * ne	Nepali
+  * nl	Dutch
+  * no	Norwegian
+  * pa	Punjabi
+  * pl	Polish
+  * pt	Portuguese
+  * ro	Romanian
+  * ru	Russian
+  * sk	Slovak
+  * sl	Slovene
+  * so	Somali
+  * sq	Albanian
+  * sv	Swedish
+  * sw	Swahili
+  * ta	Tamil
+  * te	Telugu
+  * th	Thai
+  * tl	Tagalog
+  * tr	Turkish
+  * uk	Ukrainian
+  * ur	Urdu
+  * vi	Vietnamese
+  * zh-cn	Simplified Chinese
+  * zh-tw	Traditional Chinese
+          EOF
+
+          opts.separator ""
+          opts.separator "Common options:"
+          # No argument, shows at tail.  This will print an options summary.
+          # Try it and see!
+          opts.on_tail("-h", "--help", "Show this message.") do
+            puts opts
+            exit
+          end
+        end
+      end
+
+      ##
+      # @param [String] input
+      #
+      def run(input)
+        option_parser.parse!(options[:args])
+        identifier = LanguageIdentifier.new(options)
+
+        output = identifier.run(input)
+        puts output
+      end
+
+      private
+
+      ##
+      # Shows the help message and exits the program.
+      #
+      def show_help
+        abort option_parser.to_s
+      end
+
+      ##
+      # Shows the version and exits the program.
+      #
+      def show_version
+        abort "#{option_parser.program_name} v#{VERSION} on #{RUBY_DESCRIPTION}"
+      end
+    end # CLI
+  end # LanguageIdentifier
+end # Opener

data/lib/opener/language_identifier/detector.rb

@@ -0,0 +1,36 @@
+require 'singleton'
+
+import 'org.vicomtech.opennlp.LanguageDetection.CybozuDetector'
+
+module Opener
+  class LanguageIdentifier
+    class Detector
+      attr_reader :options
+
+      include Singleton
+
+      def initialize(options={})
+        @options = options
+        @detector = CybozuDetector.new(profiles_path)
+        @semaphore = Mutex.new
+      end
+
+      def detect(input)
+        @semaphore.synchronize do
+          @detector.detect(input)
+        end
+      end
+
+      def probabilities(input)
+        @semaphore.synchronize do
+          result = @detector.detect_langs(input)
+        end
+      end
+
+      def profiles_path
+        default_path = File.expand_path("../../../../core/target/classes/profiles", __FILE__)
+        options.fetch(:profiles_path, default_path)
+      end
+    end
+  end
+end

data/lib/opener/language_identifier/kaf_builder.rb

@@ -0,0 +1,62 @@
+module Opener
+  class LanguageIdentifier
+    ##
+    # Class for building basic KAF documents that contain the correct language
+    # tag and the raw input that was anaylzed.
+    #
+    # @!attribute [r] xml
+    #  @return [Builder::XmlMarkup]
+    #
+    # @!attribute [r] original_text
+    #  @return [String]
+    #
+    # @!attribute [r] language
+    #  @return [String]
+    #
+    class KafBuilder
+      attr_reader :xml, :original_text, :language
+
+      ##
+      # @param [String] text The input text that was analyzed.
+      # @param [String] language The language of the text.
+      #
+      def initialize(text, language)
+        @xml           = Builder::XmlMarkup.new(:indent => 2)
+        @language      = language.strip
+        @original_text = text
+      end
+
+      ##
+      # Builds the KAF document.
+      #
+      def build
+        xml.instruct!(
+          :xml,
+          :version    => '1.0',
+          :encoding   => 'UTF-8',
+          :standalone => 'yes'
+        )
+
+        xml.KAF('xml:lang' => language, 'version' => version) do |node|
+          node.raw(original_text)
+        end
+      end
+
+      ##
+      # Returns the XML document as a String.
+      #
+      # @return [String]
+      #
+      def to_s
+        return xml.target!
+      end
+
+      ##
+      # @return [String]
+      #
+      def version
+        return "2.1"
+      end
+    end # KafBuilder
+  end # LanguageIdentifier
+end # Opener

data/lib/opener/language_identifier/public/markdown.css

@@ -0,0 +1,283 @@
+input[type="text"], textarea
+{
+  width: 500px;
+}
+
+body {
+  font-family: Helvetica, arial, sans-serif;
+  font-size: 14px;
+  line-height: 1.6;
+  padding-top: 10px;
+  padding-bottom: 10px;
+  background-color: white;
+  padding: 30px; }
+
+body > *:first-child {
+  margin-top: 0 !important; }
+body > *:last-child {
+  margin-bottom: 0 !important; }
+
+a {
+  color: #4183C4; }
+a.absent {
+  color: #cc0000; }
+a.anchor {
+  display: block;
+  padding-left: 30px;
+  margin-left: -30px;
+  cursor: pointer;
+  position: absolute;
+  top: 0;
+  left: 0;
+  bottom: 0; }
+
+h1, h2, h3, h4, h5, h6 {
+  margin: 20px 0 10px;
+  padding: 0;
+  font-weight: bold;
+  -webkit-font-smoothing: antialiased;
+  cursor: text;
+  position: relative; }
+
+h1:hover a.anchor, h2:hover a.anchor, h3:hover a.anchor, h4:hover a.anchor, h5:hover a.anchor, h6:hover a.anchor {
+  background: url("../../images/modules/styleguide/para.png") no-repeat 10px center;
+  text-decoration: none; }
+
+h1 tt, h1 code {
+  font-size: inherit; }
+
+h2 tt, h2 code {
+  font-size: inherit; }
+
+h3 tt, h3 code {
+  font-size: inherit; }
+
+h4 tt, h4 code {
+  font-size: inherit; }
+
+h5 tt, h5 code {
+  font-size: inherit; }
+
+h6 tt, h6 code {
+  font-size: inherit; }
+
+h1 {
+  font-size: 28px;
+  color: black; }
+
+h2 {
+  font-size: 24px;
+  border-bottom: 1px solid #cccccc;
+  color: black; }
+
+h3 {
+  font-size: 18px; }
+
+h4 {
+  font-size: 16px; }
+
+h5 {
+  font-size: 14px; }
+
+h6 {
+  color: #777777;
+  font-size: 14px; }
+
+p, blockquote, ul, ol, dl, li, table, pre {
+  margin: 15px 0; }
+
+hr {
+  background: transparent url("../../images/modules/pulls/dirty-shade.png") repeat-x 0 0;
+  border: 0 none;
+  color: #cccccc;
+  height: 4px;
+  padding: 0; }
+
+body > h2:first-child {
+  margin-top: 0;
+  padding-top: 0; }
+body > h1:first-child {
+  margin-top: 0;
+  padding-top: 0; }
+body > h1:first-child + h2 {
+  margin-top: 0;
+  padding-top: 0; }
+body > h3:first-child, body > h4:first-child, body > h5:first-child, body > h6:first-child {
+  margin-top: 0;
+  padding-top: 0; }
+
+a:first-child h1, a:first-child h2, a:first-child h3, a:first-child h4, a:first-child h5, a:first-child h6 {
+  margin-top: 0;
+  padding-top: 0; }
+
+h1 p, h2 p, h3 p, h4 p, h5 p, h6 p {
+  margin-top: 0; }
+
+li p.first {
+  display: inline-block; }
+
+ul, ol {
+  padding-left: 30px; }
+
+ul :first-child, ol :first-child {
+  margin-top: 0; }
+
+ul :last-child, ol :last-child {
+  margin-bottom: 0; }
+
+dl {
+  padding: 0; }
+dl dt {
+  font-size: 14px;
+  font-weight: bold;
+  font-style: italic;
+  padding: 0;
+  margin: 15px 0 5px; }
+dl dt:first-child {
+  padding: 0; }
+dl dt > :first-child {
+  margin-top: 0; }
+dl dt > :last-child {
+  margin-bottom: 0; }
+dl dd {
+  margin: 0 0 15px;
+  padding: 0 15px; }
+dl dd > :first-child {
+  margin-top: 0; }
+dl dd > :last-child {
+  margin-bottom: 0; }
+
+blockquote {
+  border-left: 4px solid #dddddd;
+  padding: 0 15px;
+  color: #777777; }
+blockquote > :first-child {
+  margin-top: 0; }
+blockquote > :last-child {
+  margin-bottom: 0; }
+
+table {
+  padding: 0; }
+table tr {
+  border-top: 1px solid #cccccc;
+  background-color: white;
+  margin: 0;
+  padding: 0; }
+table tr:nth-child(2n) {
+  background-color: #f8f8f8; }
+table tr th {
+  font-weight: bold;
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px; }
+table tr td {
+  border: 1px solid #cccccc;
+  text-align: left;
+  margin: 0;
+  padding: 6px 13px; }
+table tr th :first-child, table tr td :first-child {
+  margin-top: 0; }
+table tr th :last-child, table tr td :last-child {
+  margin-bottom: 0; }
+
+img {
+  max-width: 100%; }
+
+span.frame {
+  display: block;
+  overflow: hidden; }
+span.frame > span {
+  border: 1px solid #dddddd;
+  display: block;
+  float: left;
+  overflow: hidden;
+  margin: 13px 0 0;
+  padding: 7px;
+  width: auto; }
+span.frame span img {
+  display: block;
+  float: left; }
+span.frame span span {
+  clear: both;
+  color: #333333;
+  display: block;
+  padding: 5px 0 0; }
+span.align-center {
+  display: block;
+  overflow: hidden;
+  clear: both; }
+span.align-center > span {
+  display: block;
+  overflow: hidden;
+  margin: 13px auto 0;
+  text-align: center; }
+span.align-center span img {
+  margin: 0 auto;
+  text-align: center; }
+span.align-right {
+  display: block;
+  overflow: hidden;
+  clear: both; }
+span.align-right > span {
+  display: block;
+  overflow: hidden;
+  margin: 13px 0 0;
+  text-align: right; }
+span.align-right span img {
+  margin: 0;
+  text-align: right; }
+span.float-left {
+  display: block;
+  margin-right: 13px;
+  overflow: hidden;
+  float: left; }
+span.float-left span {
+  margin: 13px 0 0; }
+span.float-right {
+  display: block;
+  margin-left: 13px;
+  overflow: hidden;
+  float: right; }
+span.float-right > span {
+  display: block;
+  overflow: hidden;
+  margin: 13px auto 0;
+  text-align: right; }
+
+code, tt {
+  margin: 0 2px;
+  padding: 0 5px;
+  white-space: nowrap;
+  border: 1px solid #eaeaea;
+  background-color: #f8f8f8;
+  border-radius: 3px; }
+
+pre code {
+  margin: 0;
+  padding: 0;
+  white-space: pre;
+  border: none;
+  background: transparent; }
+
+.highlight pre {
+  background-color: #f8f8f8;
+  border: 1px solid #cccccc;
+  font-size: 13px;
+  line-height: 19px;
+  overflow: auto;
+  padding: 6px 10px;
+  border-radius: 3px; }
+
+pre {
+  background-color: #f8f8f8;
+  border: 1px solid #cccccc;
+  font-size: 13px;
+  line-height: 19px;
+  overflow: auto;
+  padding: 6px 10px;
+  border-radius: 3px; }
+pre code, pre tt {
+  background-color: transparent;
+  border: none; }
+