srx 0.1.0

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'srx'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/profile

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'srx'
+require 'optparse'
+require 'memory_profiler'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+  opts.on('-sFILE', '--srx FILE', 'SRX file (optional)')
+  opts.on('-fFORMAT', '--format FORMAT', 'Format of input text (default: text)')
+end.parse!(into: options)
+
+data = if options[:srx]
+         Srx::Data.from_file(path: options[:srx])
+       else
+         Srx::Data.default
+       end
+format = options[:format]&.to_sym || :text
+engine = Srx::Engine.new(data, format: format)
+
+license_text = File.open(File.expand_path('../LICENSE.txt', __dir__), &:read).strip.then { |t| Srx::Util.unwrap(t) }
+
+n = 100
+
+report = MemoryProfiler.report do
+  n.times { engine.segment(license_text, language: 'en') }
+end
+
+report.pretty_print

data/bin/segment

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'srx'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{$PROGRAM_NAME} [options] [file ...]"
+
+  opts.on('-sFILE', '--srx FILE', 'SRX file (optional)')
+  opts.on('-lLANGUAGE', '--language LANGUAGE', 'Language of input text (default: en)')
+end.parse!(into: options)
+
+language = options[:language] || 'en'
+data = if options[:srx]
+         Srx::Data.from_file(path: options[:srx])
+       else
+         Srx::Data.default
+       end
+engine = Srx::Engine.new(data)
+
+begin
+  text = ARGF.read
+  puts engine.segment(text, language: language)
+rescue Interrupt
+  exit(1)
+end

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/srx.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative 'srx/version'
+require_relative 'srx/data'
+require_relative 'srx/engine'
+require_relative 'srx/icu_regex'
+require_relative 'srx/util'
+require_relative 'srx/format'
+
+module Srx
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/srx/data.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Srx
+  # Abstract convenience wrapper around Nokogiri XML node
+  class XmlWrapper
+    NS = { 'srx' => 'http://www.lisa.org/srx20' }.freeze
+
+    def initialize(xml)
+      @xml = xml
+    end
+
+    def xpath(*segments)
+      path = ['.', *segments].join('/srx:')
+      @xml.xpath(path, NS)
+    end
+  end
+
+  private_constant :XmlWrapper
+
+  # SRX data
+  class Data < XmlWrapper
+    class << self
+      # Default SRX rules
+      #
+      # @return [Data]
+      def default
+        from_file(path: File.expand_path('srx-20-sample.srx', __dir__))
+      end
+
+      # @param path [String]
+      # @return [Data]
+      def from_file(path:)
+        File.open(path, &method(:from_io))
+      end
+
+      # @param io [IO]
+      # @return [Data]
+      def from_io(io)
+        new(Nokogiri::XML.parse(io))
+      end
+    end
+
+    def segment_subflows?
+      header['segmentsubflows'] == 'yes'
+    end
+
+    def cascade?
+      header['cascade'] == 'yes'
+    end
+
+    def include_start_formatting?
+      include_formatting?(:start)
+    end
+
+    def include_end_formatting?
+      include_formatting?(:end)
+    end
+
+    def include_isolated_formatting?
+      include_formatting?(:isolated)
+    end
+
+    # @return [Array<LanguageRule>]
+    def language_rules
+      @language_rules ||=
+        xpath(:srx, :body, :languagerules, :languagerule)
+        .map { |langrule| LanguageRule.new(langrule) }
+    end
+
+    # @return [Array<LanguageMap>]
+    def map_rules
+      @map_rules ||=
+        xpath(:srx, :body, :maprules, :languagemap)
+        .map { |maprule| LanguageMap.new(maprule) }
+    end
+
+    private
+
+    def header
+      @header ||= xpath(:srx, :header).first
+    end
+
+    # @param type [Symbol]
+    def format_handle(type)
+      xpath(:srx, :header, "formathandle[@type='#{type}']").first
+    end
+
+    # @param type [Symbol]
+    def include_formatting?(type)
+      elem = format_handle(type)
+
+      return elem['include'] == 'yes' if elem
+
+      # Defaults are
+      #   <formathandle type="start" include="no"/>
+      #   <formathandle type="end" include="yes"/>
+      #   <formathandle type="isolated" include="no"/>
+      case type
+      when %i[start isolated] then false
+      when :end then true
+      else raise(ArgumentError, "Unknown formatting type: #{type}")
+      end
+    end
+
+    # SRX <languagerule> element
+    class LanguageRule < XmlWrapper
+      # @return [String]
+      def name
+        @xml['languagerulename']
+      end
+
+      # @return [Array<Rule>]
+      def rules
+        @rules ||= xpath(:rule).map { |rule| Rule.new(rule) }
+      end
+
+      # SRX <rule> element
+      class Rule < XmlWrapper
+        # @return [Regexp,nil]
+        attr_reader :before_break
+
+        # @return [Regexp,nil]
+        attr_reader :after_break
+
+        def initialize(xml)
+          super(xml)
+
+          # Eagerly load everything for this class because before_break and
+          # after_break can be legitimately nil, so lazy loading gets ugly.
+
+          @break = @xml['break'].nil? || @xml['break'] == 'yes'
+
+          @before_break = xpath(:beforebreak).first&.text.then do |pattern|
+            IcuRegex.compile(pattern) if pattern
+          end
+
+          @after_break ||= xpath(:afterbreak).first&.text.then do |pattern|
+            IcuRegex.compile(pattern) if pattern
+          end
+        end
+
+        def break?
+          @break
+        end
+
+        def inspect
+          "Rule[break=#{break?},before=#{before_break},after=#{after_break}]"
+        end
+      end
+    end
+
+    # SRX <languagemap> element
+    class LanguageMap < XmlWrapper
+      # @return [String]
+      def language_rule_name
+        @xml['languagerulename']
+      end
+
+      # @return [Regexp]
+      def language_pattern
+        @language_pattern ||= @xml['languagepattern'].then do |pattern|
+          IcuRegex.compile(pattern) if pattern
+        end
+      end
+    end
+  end
+end

data/lib/srx/engine.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Srx
+  # Engine for performing SRX segmenting
+  class Engine
+    # @return [Data]
+    attr_reader :data
+
+    # @param data [Data]
+    # @param markup [Regexp]
+    def initialize(data, format: :text)
+      @data = data
+      @format = Format.get(format)
+    end
+
+    # @param str [String]
+    # @param language [String]
+    # @return [Array<String>]
+    def segment(str, language:)
+      results = []
+      rules = rules(language)
+
+      plain_text, markups = @format.extract_markups(str)
+
+      pos = 0
+      breaks_by_pos(plain_text, rules).each do |break_pos, _|
+        results << build_segment!(plain_text, markups, pos, break_pos)
+        pos = break_pos
+      end
+
+      results
+    end
+
+    # @param language [String]
+    # @return [Array<Data::Rule>]
+    def rules(language)
+      names = rule_names(language)
+
+      rule_map = @data.language_rules.map do |rule|
+        [rule.name, rule]
+      end.to_h
+
+      names.flat_map { |name| rule_map[name].rules }
+    end
+
+    # @param language [String]
+    # @return [Array<String>]
+    def rule_names(language)
+      @data.map_rules.map do |lang_map|
+        next unless lang_map.language_pattern.match?(language)
+
+        break [lang_map.language_rule_name] unless @data.cascade?
+
+        lang_map.language_rule_name
+      end.compact
+    end
+
+    # @param str [String]
+    # @param pos [Integer] the position to start searching from
+    # @param rules [Array<Data::LanguageRule::Rule>]
+    # @return [Array(Integer,Data::LanguageRule::Rule)] an array of 1) the
+    #   position of a break, and 2) the rule that matched at that position. Note
+    #   that the final break will always be at the end of the string and may not
+    #   have an associated rule.
+    def breaks_by_pos(str, rules)
+      rules
+        .flat_map { |rule| all_matches(str, rule) }
+        .group_by(&:first)
+        .transform_values { |pairs| pairs.first.last }
+        .select { |_pos, rule| rule.break? }
+        .sort_by(&:first)
+        .tap { |breaks| breaks << [str.length] unless breaks&.last&.first == str.length }
+    end
+
+    # @param str [String]
+    # @param rule [Data::LanguageRule::Rule]
+    # @return [Array<Array(Integer,Data::LanguageRule::Rule)>]
+    def all_matches(str, rule)
+      results = []
+
+      pos = 0
+      while pos < str.length
+        if rule.before_break
+          m = rule.before_break.match(str, pos)
+          break unless m
+
+          pos = m.end(0)
+          pos += 1 if pos == m.begin(0)
+
+          results << [pos, rule] if rule.after_break.nil? || m.post_match.start_with?(rule.after_break)
+        elsif rule.after_break
+          m = rule.after_break.match(str, pos)
+          break unless m
+
+          pos = m.begin(0) + 1
+          results << [pos, rule]
+        else
+          raise('Rule has neither before_break nor after_break')
+        end
+      end
+
+      results
+    end
+
+    # @param str [String]
+    # @param markups [Array<Array(Integer,String)>]
+    # @param start [Integer] start offset of segment in str
+    # @param finish [Integer] end offset of segment in str
+    def build_segment!(str, markups, start, finish)
+      segment = str[start...finish]
+
+      until markups.empty?
+        markup_pos, markup = markups.first
+        break unless start + segment.length >= markup_pos
+
+        break if start + segment.length == markup_pos && !include_edge_formatting?(markup)
+
+        segment.insert(markup_pos - start, markup)
+        markups.shift
+      end
+
+      segment
+    end
+
+    # @param markup [String]
+    # @return [Boolean] whether to include the specified edge markup in the
+    #   current segment, in accordance with <formathandle> rules
+    def include_edge_formatting?(markup)
+      return false if !@data.include_start_formatting? && @format.start_formatting?(markup)
+      return false if !@data.include_end_formatting? && @format.end_formatting?(markup)
+      return false if !@data.include_isolated_formatting? && @format.isolated_formatting?(markup)
+
+      true
+    end
+  end
+end

data/lib/srx/format.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'format/base_format'
+require_relative 'format/text'
+require_relative 'format/xml'
+
+module Srx
+  # Format-specific data and logic
+  module Format
+    FORMATS = {
+      text: Text.new,
+      xml: Xml.new,
+      html: Xml.new # TODO: specialize for HTML
+    }.freeze
+
+    class << self
+      # @param format [Symbol]
+      # @return [BaseFormat]
+      def get(format)
+        raise(ArgumentError, "Unknown format: #{format}") unless FORMATS.key?(format)
+
+        FORMATS[format]
+      end
+    end
+  end
+end

data/lib/srx/format/base_format.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Srx
+  module Format
+    # Interface definition for format support
+    class BaseFormat
+      # @abstract
+      # @param str [String]
+      # @return [Array(String,Array<Array(Integer,String)>)] two items: 1) input
+      #   +str+ with markups removed, and 2) a list of markups, i.e. +[pos,
+      #   string]+ pairs
+      def extract_markups(str)
+        raise(NotImplementedError)
+      end
+
+      # @abstract
+      # @param markup [String]
+      # @return [Boolean]
+      def start_formatting?(markup)
+        raise(NotImplementedError)
+      end
+
+      # @abstract
+      # @param markup [String]
+      # @return [Boolean]
+      def end_formatting?(markup)
+        raise(NotImplementedError)
+      end
+
+      # @abstract
+      # @param markup [String]
+      # @return [Boolean]
+      def isolated_formatting?(markup)
+        raise(NotImplementedError)
+      end
+    end
+  end
+end