srx 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 248fc2b9cd220023564c26a792536b1593b2bdefe77b71e133805658bd169d13
-  data.tar.gz: 7aa58a5e7fa2255219aaf83f7df1b09ebde3711a59c84d0cce7b3811af6f869e
+  metadata.gz: c28a49fd42454ec9968d3a0c1066af93c14ec4ebcd68fc5dccb43b24c0cf13c7
+  data.tar.gz: 268e0beffe2d9a2a9b189b9a5ce649cc026ed066d3dc168767db9d99ccdc174d
 SHA512:
-  metadata.gz: 0af9311397924bf58131fd753e879f63f32c3706998f5cbb9c3c06ab51cb626a2417d31cd52dba2d905b571d5489efc1ad088a334f7b868fdd66e8e82a64f547
-  data.tar.gz: 969da7b926a664411a887948f17ff011d8e4040ef970e71d2ecde92ff05e39039aeb2a51238326be83418fda7bdf279241145c024cf41bccc8d4d515027cc039
+  metadata.gz: 6c7edc8f3fc9895e434b20551fc1dbccb82731e6500302375db3a3ffce730fe96a78163d662da637c1455449cb459abf751f616ba38e77d4594258611831b503
+  data.tar.gz: 2e0a72a250e180ba909ac2aa4c31c0ec85bca10c4cf40979910c9fb4123e47154402b7e554dbf62a7095e623a4ac6c29188f89d6a1ee2dfce9d8a7a1c978fa12

data/.rubocop_todo.yml

@@ -1,15 +1,15 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2021-02-08 14:52:03 UTC using RuboCop version 1.9.1.
+# on 2021-02-15 14:53:27 UTC using RuboCop version 1.9.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 3
+# Offense count: 6
 # Configuration parameters: IgnoredMethods, CountRepeatedAttributes.
 Metrics/AbcSize:
-  Max: 24
+  Max: 26
 
 # Offense count: 6
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
@@ -17,17 +17,17 @@ Metrics/AbcSize:
 Metrics/BlockLength:
   Max: 269
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: IgnoredMethods.
 Metrics/CyclomaticComplexity:
   Max: 9
 
-# Offense count: 3
+# Offense count: 9
 # Configuration parameters: CountComments, CountAsOne, ExcludedMethods, IgnoredMethods.
 Metrics/MethodLength:
   Max: 25
 
-# Offense count: 1
+# Offense count: 2
 # Configuration parameters: IgnoredMethods.
 Metrics/PerceivedComplexity:
   Max: 10

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.3.0] - 2021-02-16
+
+- All `Srx::Engine` methods except `#segment` are now private
+- ICU regex syntax `\xhhhh` is now no longer converted to Ruby regex, as this
+  syntax was not correct; it now must be `\x{hhhh}`
+- ICU regex syntax `\0ooo` is now supported
+
 ## [0.2.0] - 2021-02-13
 
 - Handle HTML void elements correctly

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    srx (0.2.0)
+    srx (0.3.0)
       nokogiri (~> 1.11)
 
 GEM
@@ -36,7 +36,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
     rspec-support (3.10.2)
-    rubocop (1.9.1)
+    rubocop (1.10.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)

data/README.md

@@ -48,6 +48,20 @@ Some disadvantages:
   improve on that with better rules such as
   [LanguageTool's](https://github.com/amake/srx-languagetool-ruby).
 
+## Caveats
+
+The SRX spec calls for [ICU regular
+expressions](https://unicode-org.github.io/icu/userguide/strings/regexp.html),
+but this library uses standard [Ruby
+regexp](https://ruby-doc.org/core-2.7.0/Regexp.html). Please note:
+
+- Not all ICU syntax is supported
+- For supported syntax, in some cases the meaning of a regex may differ when
+  interpreted as Ruby regexp
+- The following ICU syntax is supported through translation to Ruby syntax:
+  - `\x{hhhh}` → `\u{hhhh}`
+  - `\0ooo` → `\u{hhhh}`
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -96,7 +110,7 @@ input = 'foo <bar baz="a. b."> bazinga'
 Srx::Engine.new(Data.default).segment(input, language: 'en')
 #=> ["foo <bar baz=\"a.", " b.\"> bazinga"]
 
-Srx::Engine.new(data, format: :xml).segment(input, language: 'en')
+Srx::Engine.new(Data.default, format: :xml).segment(input, language: 'en')
 #=> ["foo <bar baz=\"a. b.\"> bazinga"]
 ```
 

data/lib/srx/engine.rb

@@ -7,7 +7,7 @@ module Srx
     attr_reader :data
 
     # @param data [Data]
-    # @param markup [Regexp]
+    # @param format [Symbol] see {Format#get}
     def initialize(data, format: :text)
       @data = data
       @format = Format.get(format)
@@ -31,6 +31,8 @@ module Srx
       results
     end
 
+    private
+
     # @param language [String]
     # @return [Array<Data::Rule>]
     def rules(language)
@@ -58,10 +60,10 @@ module Srx
     # @param str [String]
     # @param pos [Integer] the position to start searching from
     # @param rules [Array<Data::LanguageRule::Rule>]
-    # @return [Array(Integer,Data::LanguageRule::Rule)] an array of 1) the
-    #   position of a break, and 2) the rule that matched at that position. Note
-    #   that the final break will always be at the end of the string and may not
-    #   have an associated rule.
+    # @return [Array<Array(Integer,Data::LanguageRule::Rule)>] an array of pairs
+    #   of 1) the position of a break, and 2) the rule that matched at that
+    #   position. Note that the final break will always be at the end of the
+    #   string and may not have an associated rule.
     def breaks_by_pos(str, rules)
       rules
         .flat_map { |rule| all_matches(str, rule) }

data/lib/srx/format.rb

@@ -15,7 +15,7 @@ module Srx
     }.freeze
 
     class << self
-      # @param format [Symbol]
+      # @param format [Symbol] see keys of {FORMATS} for accepted values
       # @return [BaseFormat]
       def get(format)
         raise(ArgumentError, "Unknown format: #{format}") unless FORMATS.key?(format)

data/lib/srx/icu_regex.rb

@@ -3,13 +3,20 @@
 module Srx
   # Utilities for handling SRX (ICU) regular expressions
   module IcuRegex
-    HEX_PATTERN = /(?<!\\)(?:\\\\)*\\x(?<hex>[a-f0-9]{4}|\{[a-f0-9]{4}\})/i.freeze
+    HEX_PATTERN = /(?<!\\)(?:\\\\)*\\x(?<hex>\{[a-f0-9]{1,6}\})/i.freeze
+    OCTAL_PATTERN = /(?<!\\)(?:\\\\)*\\0(?<oct>[0-7]{1,3})/i.freeze
 
     class << self
       # @param icu_regex [String]
       # @return [String]
       def to_ruby(icu_regex)
-        icu_regex.gsub(HEX_PATTERN, '\u\k<hex>')
+        result = icu_regex.dup
+        result.gsub!(HEX_PATTERN, '\u\k<hex>')
+        result.gsub!(OCTAL_PATTERN) do |m|
+          $LAST_MATCH_INFO['oct'].to_i(8).then { |o| o <= 255 ? format(%q(\u{%x}), o) : m }
+        end
+
+        result
       end
 
       # @param icu_regex [String]

data/lib/srx/srx-20-sample.srx

@@ -68,7 +68,7 @@
                 \xff01: Fullwidth exclamation mark
                 -->
                 <rule break="yes">
-                    <beforebreak>[\xff61\x3002\xff0e\xff1f\xff01]+</beforebreak>
+                    <beforebreak>[\x{ff61}\x{3002}\x{ff0e}\x{ff1f}\x{ff01}]+</beforebreak>
                     <afterbreak></afterbreak>
                 </rule>
             </languagerule>

data/lib/srx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Srx
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: srx
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Aaron Madlon-Kay
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri