srx 0.1.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18c2fc1da5f4a792393a9dd4955c79d73bc9d00943000f38689a5763d8c8ff4c
-  data.tar.gz: 2385b9f65e61ad291e1419e217b737653bbeea0d4f62dc7fbb4ae35e6948984c
+  metadata.gz: f678f4fe5e7f30edbe600f27a5dd00217528d84291248ebf76f8d5c4df62f333
+  data.tar.gz: 5d096e1688a5c57756bb86be9a31d504aa99cbb877de434737e5c4578808f4f1
 SHA512:
-  metadata.gz: 915b837e6a239f7de688d51ca3fca2bbdcf898234f825cbd49894223e8c31f71167d6a22d9097a36ab75ea3fc8f383ae790d3df8e0e3fce2fb8210022412fad2
-  data.tar.gz: 326d3de2c328aa49be07c7df62229fe40c8cf3b2c4101f090b9c805b9e3a2fcd1c8eec155b6b3af9c556ff5c548d5263425975bdc9b0368b334342ce47dd71d9
+  metadata.gz: 78d28e9ebb51c4ac61f2b15dc27d527f802a7a9c40812741ca817a14b0e231e0dce310f0b9c38411fb8dfe2f02953ca7344a90ad85fea5290d2b234be5795130
+  data.tar.gz: 2521273e4f164b44be0f43b2dd54cc17f380edc130e600c37cf8ef2b0ae847e287e91a59c036e638461462c7b2fc83401a5c348501429fe107becacbdd906188

data/.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
         ruby-version: 2.7.2
     - name: Install
       run: |
-        gem install bundler -v 2.2.7
+        gem install bundler -v 2.2.9
         bundle install
     - name: Type check
       run: bundle exec solargraph typecheck --level typed

data/.rubocop.yml

@@ -1,5 +1,9 @@
 inherit_from: .rubocop_todo.yml
 
+inherit_mode:
+  merge:
+    - Exclude
+
 AllCops:
   TargetRubyVersion: 2.4
   SuggestExtensions: false
@@ -10,4 +14,4 @@ AllCops:
 Layout/LineLength:
   Max: 120
   Exclude:
-    - 'test/srx/golden_rules_test.rb'
+    - 'test/golden_rules_test.rb'

data/.rubocop_todo.yml

@@ -1,12 +1,12 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-02-08 14:52:03 UTC using RuboCop version 1.9.1.
+# on 2021-02-25 04:43:35 UTC using RuboCop version 1.10.0.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 3
+# Offense count: 5
 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
 Metrics/AbcSize:
   Max: 24
@@ -17,15 +17,15 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 269
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:
   Max: 9
 
-# Offense count: 3
+# Offense count: 9
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 Metrics/MethodLength:
-  Max: 25
+  Max: 26
 
 # Offense count: 1
 # Configuration parameters: IgnoredMethods.

data/CHANGELOG.md

@@ -0,0 +1,29 @@
+## [Unreleased]
+
+## [0.6.0] - 2021-04-15
+
+- Improved HTML parsing accuracy
+
+## [0.5.0] - 2021-02-25
+
+- When `nil` is supplied for the `language` parameter, it is now treated as the
+  empty string for rule-matching purposes (previously it would match no rules)
+
+## [0.4.0] - 2021-02-18
+
+- Optimize memory usage
+
+## [0.3.0] - 2021-02-16
+
+- All `Srx::Engine` methods except `#segment` are now private
+- ICU regex syntax `\xhhhh` is now no longer converted to Ruby regex, as this
+  syntax was not correct; it now must be `\x{hhhh}`
+- ICU regex syntax `\0ooo` is now supported
+
+## [0.2.0] - 2021-02-13
+
+- Handle HTML void elements correctly
+
+## [0.1.0] - 2021-02-13
+
+- Initial release

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    srx (0.1.0)
+    srx (0.6.0)
       nokogiri (~> 1.11)
 
 GEM
@@ -14,29 +14,29 @@ GEM
     diff-lcs (1.4.4)
     e2mmap (0.1.0)
     jaro_winkler (1.5.4)
-    kramdown (2.3.0)
+    kramdown (2.3.1)
       rexml
     kramdown-parser-gfm (1.1.0)
       kramdown (~> 2.0)
     memory_profiler (1.0.0)
-    minitest (5.14.3)
-    nokogiri (1.11.1-x86_64-darwin)
+    minitest (5.14.4)
+    nokogiri (1.11.3-x86_64-darwin)
       racc (~> 1.4)
     parallel (1.20.1)
-    parser (3.0.0.0)
+    parser (3.0.1.0)
       ast (~> 2.4.1)
     racc (1.5.2)
     rainbow (3.0.0)
     rake (13.0.3)
-    regexp_parser (2.0.3)
+    regexp_parser (2.1.1)
     reverse_markdown (2.0.0)
       nokogiri
-    rexml (3.2.4)
+    rexml (3.2.5)
     rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.9.1)
+    rubocop (1.12.1)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
@@ -48,7 +48,7 @@ GEM
     rubocop-ast (1.4.1)
       parser (>= 2.7.1.5)
     ruby-progressbar (1.11.0)
-    solargraph (0.40.3)
+    solargraph (0.40.4)
       backport (~> 1.1)
       benchmark
       bundler (>= 1.17.2)
@@ -81,4 +81,4 @@ DEPENDENCIES
   srx!
 
 BUNDLED WITH
-   2.2.7
+   2.2.9

data/README.md

@@ -13,7 +13,10 @@ This gem provides facilities for reading SRX files and an engine for performing
 segmentation.
 
 Only a minimal rule set is supplied by default; for actual usage you are
-encouraged to supply your own SRX rules.
+encouraged to supply your own SRX rules. One such set of rules is that from
+[LanguageTool](https://languagetool.org/); this is conveniently packaged into a
+companion gem:
+[srx-languagetool-ruby](https://github.com/amake/srx-languagetool-ruby).
 
 ## What's different about this gem?
 
@@ -43,7 +46,21 @@ Some disadvantages:
   test](https://github.com/diasks2/pragmatic_segmenter#comparison-of-segmentation-tools-libraries-and-algorithms),
   scoring 47% (English) and 48% (others) with the default rules. However you can
   improve on that with better rules such as
-  [LanguageTool's](https://github.com/languagetool-org/languagetool/blob/05707300df14668e97d064811931e0668f2b695b/languagetool-core/src/main/resources/org/languagetool/resource/segment.srx).
+  [LanguageTool's](https://github.com/amake/srx-languagetool-ruby).
+
+## Caveats
+
+The SRX spec calls for [ICU regular
+expressions](https://unicode-org.github.io/icu/userguide/strings/regexp.html),
+but this library uses standard [Ruby
+regexp](https://ruby-doc.org/core-2.7.0/Regexp.html). Please note:
+
+- Not all ICU syntax is supported
+- For supported syntax, in some cases the meaning of a regex may differ when
+  interpreted as Ruby regexp
+- The following ICU syntax is supported through translation to Ruby syntax:
+  - `\x{hhhh}` → `\u{hhhh}`
+  - `\0ooo` → `\u{hhhh}`
 
 ## Installation
 
@@ -93,7 +110,7 @@ input = 'foo <bar baz="a. b."> bazinga'
 Srx::Engine.new(Data.default).segment(input, language: 'en')
 #=> ["foo <bar baz=\"a.", " b.\"> bazinga"]
 
-Srx::Engine.new(data, format: :xml).segment(input, language: 'en')
+Srx::Engine.new(Data.default, format: :xml).segment(input, language: 'en')
 #=> ["foo <bar baz=\"a. b.\"> bazinga"]
 ```
 

data/lib/srx/data.rb

@@ -43,23 +43,23 @@ module Srx
     end
 
     def segment_subflows?
-      header['segmentsubflows'] == 'yes'
+      @segment_subflows ||= header['segmentsubflows'] == 'yes'
     end
 
     def cascade?
-      header['cascade'] == 'yes'
+      @cascade ||= header['cascade'] == 'yes'
     end
 
     def include_start_formatting?
-      include_formatting?(:start)
+      @include_start_formatting ||= include_formatting?(:start)
     end
 
     def include_end_formatting?
-      include_formatting?(:end)
+      @include_end_formatting ||= include_formatting?(:end)
     end
 
     def include_isolated_formatting?
-      include_formatting?(:isolated)
+      @include_isolated_formatting ||= include_formatting?(:isolated)
     end
 
     # @return [Array<LanguageRule>]
@@ -108,7 +108,7 @@ module Srx
     class LanguageRule < XmlWrapper
       # @return [String]
       def name
-        @xml['languagerulename']
+        @name ||= @xml['languagerulename']
       end
 
       # @return [Array<Rule>]
@@ -130,13 +130,13 @@ module Srx
           # Eagerly load everything for this class because before_break and
           # after_break can be legitimately nil, so lazy loading gets ugly.
 
-          @break = @xml['break'].nil? || @xml['break'] == 'yes'
+          @break = @xml['break'].then { |brk| brk.nil? || brk == 'yes' }
 
           @before_break = xpath(:beforebreak).first&.text.then do |pattern|
             IcuRegex.compile(pattern) if pattern
           end
 
-          @after_break ||= xpath(:afterbreak).first&.text.then do |pattern|
+          @after_break = xpath(:afterbreak).first&.text.then do |pattern|
             IcuRegex.compile(pattern) if pattern
           end
         end
@@ -155,7 +155,7 @@ module Srx
     class LanguageMap < XmlWrapper
       # @return [String]
       def language_rule_name
-        @xml['languagerulename']
+        @language_rule_name ||= @xml['languagerulename']
       end
 
       # @return [Regexp]

data/lib/srx/engine.rb

@@ -7,7 +7,7 @@ module Srx
     attr_reader :data
 
     # @param data [Data]
-    # @param markup [Regexp]
+    # @param format [Symbol] see {Format#get}
     def initialize(data, format: :text)
       @data = data
       @format = Format.get(format)
@@ -31,6 +31,8 @@ module Srx
       results
     end
 
+    private
+
     # @param language [String]
     # @return [Array<Data::Rule>]
     def rules(language)
@@ -43,9 +45,11 @@ module Srx
       names.flat_map { |name| rule_map[name].rules }
     end
 
-    # @param language [String]
+    # @param language [String] nil treated as empty string
     # @return [Array<String>]
     def rule_names(language)
+      language ||= ''
+
       @data.map_rules.map do |lang_map|
         next unless lang_map.language_pattern.match?(language)
 
@@ -56,20 +60,19 @@ module Srx
     end
 
     # @param str [String]
-    # @param pos [Integer] the position to start searching from
     # @param rules [Array<Data::LanguageRule::Rule>]
-    # @return [Array(Integer,Data::LanguageRule::Rule)] an array of 1) the
-    #   position of a break, and 2) the rule that matched at that position. Note
-    #   that the final break will always be at the end of the string and may not
-    #   have an associated rule.
+    # @return [Array<Array(Integer,Data::LanguageRule::Rule)>] an array of pairs
+    #   of 1) the position of a break, and 2) the rule that matched at that
+    #   position. Note that the final break will always be at the end of the
+    #   string and may not have an associated rule.
     def breaks_by_pos(str, rules)
-      rules
-        .flat_map { |rule| all_matches(str, rule) }
-        .group_by(&:first)
-        .transform_values { |pairs| pairs.first.last }
-        .select { |_pos, rule| rule.break? }
-        .sort_by(&:first)
-        .tap { |breaks| breaks << [str.length] unless breaks&.last&.first == str.length }
+      grouped = rules.flat_map { |rule| all_matches(str, rule) }
+                     .group_by(&:first)
+      grouped.transform_values! { |pairs| pairs.first.last }
+      grouped.select! { |_pos, rule| rule.break? }
+      result = grouped.sort_by(&:first)
+      result << [str.length] unless result.last&.first == str.length
+      result
     end
 
     # @param str [String]

data/lib/srx/format.rb

@@ -3,6 +3,7 @@
 require_relative 'format/base_format'
 require_relative 'format/text'
 require_relative 'format/xml'
+require_relative 'format/html'
 
 module Srx
   # Format-specific data and logic
@@ -10,11 +11,11 @@ module Srx
     FORMATS = {
       text: Text.new,
       xml: Xml.new,
-      html: Xml.new # TODO: specialize for HTML
+      html: Html.new
     }.freeze
 
     class << self
-      # @param format [Symbol]
+      # @param format [Symbol] see keys of {FORMATS} for accepted values
       # @return [BaseFormat]
       def get(format)
         raise(ArgumentError, "Unknown format: #{format}") unless FORMATS.key?(format)

data/lib/srx/format/html.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'English'
+
+module Srx
+  module Format
+    # Support for HTML. Tag grammar based on XML.
+    #
+    # @see https://www.w3.org/TR/xml/
+    # @see https://html.spec.whatwg.org/multipage/syntax.html
+    class Html < Xml
+      # Differs from XML in supporting unquoted values
+      # @see https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
+      ATT_VALUE = /#{Xml::ATT_VALUE}|(?:[^<>&"'`=\u0020\u0009\u000D\u000A]|#{Xml::REFERENCE})+/.freeze
+
+      # Differs from XML in supporting empty attributes
+      # @see https://html.spec.whatwg.org/multipage/syntax.html#attributes-2
+      ATTRIBUTE = /#{Xml::NAME}(?:#{Xml::EQUALS}#{ATT_VALUE})?/.freeze
+
+      START_TAG = /<(?<name>#{Xml::NAME})(?:#{Xml::SPACE}#{ATTRIBUTE})*#{Xml::SPACE}?>/.freeze
+      EMPTY_ELEM_TAG = %r{<#{Xml::NAME}(?:#{Xml::SPACE}#{ATTRIBUTE})*#{Xml::SPACE}?/>}.freeze
+
+      TAG = /#{START_TAG}|#{Xml::END_TAG}|#{EMPTY_ELEM_TAG}/.freeze
+
+      # A set of HTML tags that are "void elements", meaning they do not need a
+      # paired closing tag.
+      #
+      # @see https://html.spec.whatwg.org/#void-elements
+      # @see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/command
+      # @see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/keygen
+      # @see https://developer.mozilla.org/en-US/docs/Web/HTML/Element/menuitem
+      VOID_ELEMENTS = Set[
+        'area', 'base', 'br', 'col', 'command', 'embed', 'hr', 'img', 'input',
+        'link', 'meta', 'menuitem', 'param', 'source', 'track', 'wbr'
+      ].freeze
+
+      def extract_markups(str)
+        extract_markups_by_pattern(str, TAG)
+      end
+
+      def start_formatting?(markup)
+        START_TAG.match(markup) do |m|
+          !VOID_ELEMENTS.include?(m['name'])
+        end
+      end
+
+      def isolated_formatting?(markup)
+        return true if EMPTY_ELEM_TAG.match?(markup)
+
+        START_TAG.match(markup) do |m|
+          VOID_ELEMENTS.include?(m['name'])
+        end
+      end
+    end
+  end
+end

data/lib/srx/format/xml.rb

@@ -27,14 +27,7 @@ module Srx
       TAG = /#{START_TAG}|#{END_TAG}|#{EMPTY_ELEM_TAG}/.freeze
 
       def extract_markups(str)
-        markups = []
-
-        plain_text = str.gsub(TAG) do |match|
-          markups << [$LAST_MATCH_INFO.begin(0), match]
-          ''
-        end
-
-        [plain_text, markups]
+        extract_markups_by_pattern(str, TAG)
       end
 
       def start_formatting?(markup)
@@ -48,6 +41,21 @@ module Srx
       def isolated_formatting?(markup)
         EMPTY_ELEM_TAG.match?(markup)
       end
+
+      protected
+
+      # @param str [String]
+      # @param pattern [Regexp]
+      def extract_markups_by_pattern(str, pattern)
+        markups = []
+
+        plain_text = str.gsub(pattern) do |match|
+          markups << [$LAST_MATCH_INFO.begin(0), match]
+          ''
+        end
+
+        [plain_text, markups]
+      end
     end
   end
 end

data/lib/srx/icu_regex.rb

@@ -3,13 +3,20 @@
 module Srx
   # Utilities for handling SRX (ICU) regular expressions
   module IcuRegex
-    HEX_PATTERN = /(?<!\\)(?:\\\\)*\\x(?<hex>[a-f0-9]{4}|\{[a-f0-9]{4}\})/i.freeze
+    HEX_PATTERN = /(?<!\\)(?:\\\\)*\\x(?<hex>\{[a-f0-9]{1,6}\})/i.freeze
+    OCTAL_PATTERN = /(?<!\\)(?:\\\\)*\\0(?<oct>[0-7]{1,3})/i.freeze
 
     class << self
       # @param icu_regex [String]
       # @return [String]
       def to_ruby(icu_regex)
-        icu_regex.gsub(HEX_PATTERN, '\u\k<hex>')
+        result = icu_regex.dup
+        result.gsub!(HEX_PATTERN, '\u\k<hex>')
+        result.gsub!(OCTAL_PATTERN) do |m|
+          $LAST_MATCH_INFO['oct'].to_i(8).then { |o| o <= 255 ? format(%q(\u{%x}), o) : m }
+        end
+
+        result
       end
 
       # @param icu_regex [String]

data/lib/srx/srx-20-sample.srx

@@ -68,7 +68,7 @@
                 \xff01: Fullwidth exclamation mark
                 -->
                 <rule break="yes">
-                    <beforebreak>[\xff61\x3002\xff0e\xff1f\xff01]+</beforebreak>
+                    <beforebreak>[\x{ff61}\x{3002}\x{ff0e}\x{ff1f}\x{ff01}]+</beforebreak>
                     <afterbreak></afterbreak>
                 </rule>
             </languagerule>

data/lib/srx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Srx
-  VERSION = '0.1.0'
+  VERSION = '0.6.0'
 end

data/srx.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/amake/srx-ruby.git'
+  spec.metadata['changelog_uri'] = 'https://github.com/amake/srx-ruby/blob/master/CHANGELOG.md'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: srx
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Aaron Madlon-Kay
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -135,6 +135,7 @@ files:
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
 - ".solargraph.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
@@ -150,6 +151,7 @@ files:
 - lib/srx/engine.rb
 - lib/srx/format.rb
 - lib/srx/format/base_format.rb
+- lib/srx/format/html.rb
 - lib/srx/format/text.rb
 - lib/srx/format/xml.rb
 - lib/srx/icu_regex.rb
@@ -163,6 +165,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/amake/srx-ruby
   source_code_uri: https://github.com/amake/srx-ruby.git
+  changelog_uri: https://github.com/amake/srx-ruby/blob/master/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths: