srx 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 18c2fc1da5f4a792393a9dd4955c79d73bc9d00943000f38689a5763d8c8ff4c
+  data.tar.gz: 2385b9f65e61ad291e1419e217b737653bbeea0d4f62dc7fbb4ae35e6948984c
+SHA512:
+  metadata.gz: 915b837e6a239f7de688d51ca3fca2bbdcf898234f825cbd49894223e8c31f71167d6a22d9097a36ab75ea3fc8f383ae790d3df8e0e3fce2fb8210022412fad2
+  data.tar.gz: 326d3de2c328aa49be07c7df62229fe40c8cf3b2c4101f090b9c805b9e3a2fcd1c8eec155b6b3af9c556ff5c548d5263425975bdc9b0368b334342ce47dd71d9

data/.dir-locals.el

@@ -0,0 +1,4 @@
+;;; Directory Local Variables
+;;; For more information see (info "(emacs) Directory Variables")
+
+((ruby-mode . ((lsp-solargraph-use-bundler . t))))

data/.github/workflows/main.yml

@@ -0,0 +1,21 @@
+name: Ruby
+
+on: [push,pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.7.2
+    - name: Install
+      run: |
+        gem install bundler -v 2.2.7
+        bundle install
+    - name: Type check
+      run: bundle exec solargraph typecheck --level typed
+    - name: Unit tests
+      run: bundle exec rake

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor
+
+.byebug_history

data/.rubocop.yml

@@ -0,0 +1,13 @@
+inherit_from: .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.4
+  SuggestExtensions: false
+  NewCops: enable
+  Exclude:
+    - '**/*~'
+
+Layout/LineLength:
+  Max: 120
+  Exclude:
+    - 'test/srx/golden_rules_test.rb'

data/.rubocop_todo.yml

@@ -0,0 +1,33 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2021-02-08 14:52:03 UTC using RuboCop version 1.9.1.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 3
+# Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
+Metrics/AbcSize:
+  Max: 24
+
+# Offense count: 6
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+# IgnoredMethods: refine
+Metrics/BlockLength:
+  Max: 269
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/CyclomaticComplexity:
+  Max: 9
+
+# Offense count: 3
+# Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
+Metrics/MethodLength:
+  Max: 25
+
+# Offense count: 1
+# Configuration parameters: IgnoredMethods.
+Metrics/PerceivedComplexity:
+  Max: 10

data/.solargraph.yml

@@ -0,0 +1,17 @@
+---
+include:
+- "**/*.rb"
+exclude:
+- spec/**/*
+- test/**/*
+- vendor/**/*
+- ".bundle/**/*"
+require: []
+domains: []
+reporters:
+- rubocop
+- require_not_found
+- typecheck:typed
+require_paths: []
+plugins: []
+max_files: 5000

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in srx.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,84 @@
+PATH
+  remote: .
+  specs:
+    srx (0.1.0)
+      nokogiri (~> 1.11)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    backport (1.1.2)
+    benchmark (0.1.1)
+    byebug (11.1.3)
+    diff-lcs (1.4.4)
+    e2mmap (0.1.0)
+    jaro_winkler (1.5.4)
+    kramdown (2.3.0)
+      rexml
+    kramdown-parser-gfm (1.1.0)
+      kramdown (~> 2.0)
+    memory_profiler (1.0.0)
+    minitest (5.14.3)
+    nokogiri (1.11.1-x86_64-darwin)
+      racc (~> 1.4)
+    parallel (1.20.1)
+    parser (3.0.0.0)
+      ast (~> 2.4.1)
+    racc (1.5.2)
+    rainbow (3.0.0)
+    rake (13.0.3)
+    regexp_parser (2.0.3)
+    reverse_markdown (2.0.0)
+      nokogiri
+    rexml (3.2.4)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.9.1)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.4.1)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.11.0)
+    solargraph (0.40.3)
+      backport (~> 1.1)
+      benchmark
+      bundler (>= 1.17.2)
+      e2mmap
+      jaro_winkler (~> 1.5)
+      kramdown (~> 2.3)
+      kramdown-parser-gfm (~> 1.1)
+      parser (~> 3.0)
+      reverse_markdown (>= 1.0.5, < 3)
+      rubocop (>= 0.52)
+      thor (~> 1.0)
+      tilt (~> 2.0)
+      yard (~> 0.9, >= 0.9.24)
+    thor (1.1.0)
+    tilt (2.0.10)
+    unicode-display_width (2.0.0)
+    yard (0.9.26)
+
+PLATFORMS
+  x86_64-darwin-20
+
+DEPENDENCIES
+  byebug
+  memory_profiler
+  minitest
+  rake
+  rspec-expectations
+  rubocop
+  solargraph
+  srx!
+
+BUNDLED WITH
+   2.2.7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2021 Aaron Madlon-Kay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,120 @@
+# SRX for Ruby
+
+SRX is a specification for segmenting text, i.e. splitting text into sentences.
+More specifically it is
+
+- An XML-based format for specifying segmentation rules, and
+- An algorithm by which the rules are applied
+
+See the [SRX 2.0 Specification](http://www.ttt.org/oscarStandards/srx/srx20.html)
+for full details.
+
+This gem provides facilities for reading SRX files and an engine for performing
+segmentation.
+
+Only a minimal rule set is supplied by default; for actual usage you are
+encouraged to supply your own SRX rules.
+
+## What's different about this gem?
+
+There are lots of good segmentation gems out there such as
+
+- [pragmatic_segmenter](https://github.com/diasks2/pragmatic_segmenter)
+- [TactfulTokenizer](https://github.com/zencephalon/Tactful_Tokenizer)
+- [Punkt](https://github.com/lfcipriani/punkt-segmenter)
+
+What makes SRX different is:
+
+- It allows easy customization and exchange of rules via SRX files
+- It preserves whitespace surrounding break points
+- It offers advanced XML/HTML tag handling: it won't be fooled by false breaks
+  in e.g. attribute values
+
+Some other advantages that are not unique to SRX:
+
+- It is offered under a very permissive license
+- It is relatively lightweight as a dependency
+- It is fast (though this depends somewhat on the ruleset you use)
+
+Some disadvantages:
+
+- It is inherently rule-based, with all of the weaknesses that implies
+- It is not very accurate on the [Golden Rules
+  test](https://github.com/diasks2/pragmatic_segmenter#comparison-of-segmentation-tools-libraries-and-algorithms),
+  scoring 47% (English) and 48% (others) with the default rules. However you can
+  improve on that with better rules such as
+  [LanguageTool's](https://github.com/languagetool-org/languagetool/blob/05707300df14668e97d064811931e0668f2b695b/languagetool-core/src/main/resources/org/languagetool/resource/segment.srx).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'srx'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install srx
+
+## Usage
+
+Use the default rules like so. Specify the language according the `<maprules>`
+of your SRX (usually two-letter [ISO 639-1
+codes](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes)).
+
+```ruby
+require 'srx'
+
+data = Srx::Data.default
+engine = Srx::Engine.new(data)
+engine.segment('Hi. How are you?', language: 'en') #=> ["Hi.", " How are you?"]
+```
+
+Or bring your own rules:
+
+```ruby
+data = Srx::Data.from_file(path: 'path/to/my/rules.srx')
+engine = Srx::Engine.new(data)
+```
+
+Specify the format as `:xml` or `:html` to benefit from special handling of
+tags:
+
+```ruby
+# This should only be one segment, but handling as plain text incorrectly
+# produces two segments.
+input = 'foo <bar baz="a. b."> bazinga'
+
+Srx::Engine.new(Data.default).segment(input, language: 'en')
+#=> ["foo <bar baz=\"a.", " b.\"> bazinga"]
+
+Srx::Engine.new(data, format: :xml).segment(input, language: 'en')
+#=> ["foo <bar baz=\"a. b.\"> bazinga"]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake test` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and the created tag, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/amake/srx.
+
+## License
+
+The gem is available as open source under the terms of the [MIT
+License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/bin/benchmark

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'srx'
+require 'optparse'
+require 'benchmark'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-sFILE', '--srx FILE', 'SRX file (optional)')
+  opts.on('-fFORMAT', '--format FORMAT', 'Format of input text (default: text)')
+end.parse!(into: options)
+
+data = if options[:srx]
+         Srx::Data.from_file(path: options[:srx])
+       else
+         Srx::Data.default
+       end
+format = options[:format]&.to_sym || :text
+engine = Srx::Engine.new(data, format: format)
+
+license_text = File.open(File.expand_path('../LICENSE.txt', __dir__), &:read).strip.then { |t| Srx::Util.unwrap(t) }
+
+# Golden Rules speed test; see
+# https://github.com/diasks2/pragmatic_segmenter#speed-performance-benchmarks
+gold_text = Srx::Util.unwrap(<<~TXT)
+  Hello World. My name is Jonas.
+  What is your name? My name is Jonas.
+  There it is! I found it.
+  My name is Jonas E. Smith.
+  Please turn to p. 55.
+  Were Jane and co. at the party?
+  They closed the deal with Pitt, Briggs & Co. at noon.
+  Let's ask Jane and co. They should know.
+  They closed the deal with Pitt, Briggs & Co. It closed yesterday.
+  I can see Mt. Fuji from here.
+  St. Michael's Church is on 5th st. near the light.
+  That is JFK Jr.'s book.
+  I visited the U.S.A. last year.
+  I live in the E.U. How about you?
+  I live in the U.S. How about you?
+  I work for the U.S. Government in Virginia.
+  I have lived in the U.S. for 20 years.
+  At 5 a.m. Mr. Smith went to the bank. He left the bank at 6 P.M. Mr. Smith then went to the store.
+  She has $100.00 in her bag.
+  She has $100.00. It is in her bag.
+  He teaches science (He previously worked for 5 years as an engineer.) at the local University.
+  Her email is Jane.Doe@example.com. I sent her an email.
+  The site is: https://www.example.50.com/new-site/awesome_content.html. Please check it out.
+  She turned to him, 'This is great.' she said.
+  She turned to him, "This is great." she said.
+  She turned to him, "This is great." She held the book out to show him.
+  Hello!! Long time no see.
+  Hello?? Who is there?
+  Hello!? Is that you?
+  Hello?! Is that you?
+  1.) The first item 2.) The second item
+  1.) The first item. 2.) The second item.
+  1) The first item 2) The second item
+  1) The first item. 2) The second item.
+  1. The first item 2. The second item
+  1. The first item. 2. The second item.
+  • 9. The first item • 10. The second item
+  ⁃9. The first item ⁃10. The second item
+  a. The first item b. The second item c. The third list item
+  This is a sentence
+  cut off in the middle because pdf.
+  It was a cold#{' '}
+  night in the city.
+  features
+  contact manager
+  events, activities
+
+  You can find it at N°. 1026.253.553. That is where the treasure is.
+  She works at Yahoo! in the accounting department.
+  We make a good team, you and I. Did you see Albert I. Jones yesterday?
+  Thoreau argues that by simplifying one’s life, “the laws of the universe will appear less complex. . . .”
+  "Bohr [...] used the analogy of parallel stairways [...]" (Smith 55).
+  If words are left off at the end of a sentence, and that is all that is omitted, indicate the omission with ellipsis marks (preceded and followed by a space) and then indicate the end of the sentence with a period . . . . Next sentence.
+  I never meant that.... She left the store.
+  I wasn’t really ... well, what I mean...see . . . what I'm saying, the thing is . . . I didn’t mean it.
+  One further habit which was somewhat weakened . . . was that of combining words into self-interpreting compounds. . . . The practice was not abandoned. . . .
+TXT
+
+n = 100
+
+Benchmark.bm do |x|
+  x.report('LICENSE.txt (en)') { n.times { engine.segment(license_text, language: 'en') } }
+  x.report('LICENSE.txt (zz)') { n.times { engine.segment(license_text, language: 'zz') } }
+  x.report('Golden Rules (en)') { n.times { engine.segment(gold_text, language: 'en') } }
+end