gsm_encoder 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cc5e53ca18eacc9115c7395a5f6c833f22571d19
-  data.tar.gz: eab20ccf00096deef3577367472e87ae88dfa6c0
+SHA256:
+  metadata.gz: 328cd3d4611243a39035910afc138fda9bdcc23620edc2c6fa83eb4bf9bfa4dc
+  data.tar.gz: 450843ca2309b3070ed92835b5e81620c7bf38467c8e435017e5692c3e8cce7c
 SHA512:
-  metadata.gz: afd5459dc5be3e784c6b691d1ab6e6600cc0f333965cd1ad87b068a28787b98c92abc75534078f60e79e37506c0d8d2cbe5afe940032115949733593e71ba814
-  data.tar.gz: 088b1bbe74116558fd69645a61f8a4bc8df9f7f501f893071f733165002c6774ef6f6a30502ed728b4b9e9a1b3604623577c4abec30080b70bf331b3fc443c9f
+  metadata.gz: 98bd3e1a50f2e05b957cff8c041666e4647d8e9c0ef28ad795ef610fa5423e9978d9015b318a9b116e6dd4b65deea03b0ed8298c7c04eba50452ec59ce43aece
+  data.tar.gz: e2b7a8daacf2c746946fc772b0eb22aa344de19e811322c637c24d810cb50c455db399b2b1dc10c55371c533d2f46726700aac148701d55faa0d08eb76421eea

data/README.md

@@ -29,6 +29,35 @@ __NOTE: ruby >= 1.9.2 is required__
     # replaces unsupported chars with a provided value ' '
     GSMEncoder.encode('`', ' ') # => ' '
 
+## Extension tables
+
+Characters like `{ } [ ] ~ \ | € ^` are sent as an escape plus a code from an
+extension table. Two tables are built in:
+
+- `:spanish` (the default) - the Spanish national language single shift table.
+  It adds `ç Á Í Ó Ú á í ó ú` on top of the standard table.
+- `:standard` - the GSM 03.38 default extension table.
+
+The Spanish characters are only valid GSM 7-bit when the message header selects
+the Spanish shift table. Most SMS providers (Twilio, Telnyx, Plivo, ...) don't,
+so a message containing them is sent as UCS-2 instead. That cuts a segment from
+160 characters to 70. Use `:standard` if you rely on `can_encode?` or
+`encode` to keep messages in GSM 7-bit:
+
+    # for every call
+    GSMEncoder.extension_table = :standard
+
+    # or for one call
+    GSMEncoder.can_encode?('á', extension_table: :standard) # => false
+    GSMEncoder.encode('á', extension_table: :standard) # => '?'
+    GSMEncoder.decode(bytes, extension_table: :standard)
+
+You can also register your own table, an Array indexed by GSM code with `nil`
+for unused codes:
+
+    GSMEncoder.register_extension_table(:mine, table)
+    GSMEncoder.extension_table = :mine
+
 ## Updates
 
 ### 0.1.1
@@ -51,3 +80,14 @@ Code cleanup and speedup
 ### 0.1.5
 
 Even faster. About 4 times. Many thanks to @dlarrabee
+
+### 0.1.7
+
+Even faster! Many thanks to @ThomasSevestre
+
+### 0.1.8
+
+- Configurable extension table (`:standard` or `:spanish`), defaults to `:spanish`
+- Adds form feed to the extension tables
+- `decode` returns the replacement for unknown codes instead of raising `TypeError`
+

data/lib/gsm_encoder.rb

@@ -1,24 +1,34 @@
-#encoding: utf-8
+# frozen_string_literal: true
 # Stealing from Twitter's Java implementation
 # https://github.com/twitter/cloudhopper-commons-charset/blob/master/src/main/java/com/cloudhopper/commons/charset/GSMCharset.java
 
 #
 # This class encodes and decodes Ruby Strings to and from the SMS default
-# alphabet. It also supports the default extension table. The default alphabet
-# and it's extension table is defined in GSM 03.38.
+# alphabet. It also supports an extension table. The default alphabet and
+# its extension tables are defined in GSM 03.38 (3GPP TS 23.038).
+#
+# Which extension table is used can be configured:
+#
+#   GSMEncoder.extension_table = :standard # GSM 03.38 default extension table only
+#   GSMEncoder.extension_table = :spanish  # Spanish single shift table (the default)
+#
+# or per call:
+#
+#   GSMEncoder.encode("¿Qué?", extension_table: :standard)
 module GSMEncoder
-  DEFAULT_REPLACE_CHAR = "?"
+  # GSM to UTF8 tables
+  nl = 10.chr
+  ff = 12.chr
+  cr = 13.chr
+  bs = 92.chr
 
-  EXTENDED_ESCAPE = 0x1b
-  NL = 10.chr
-  CR = 13.chr
-  BS = 92.chr
+  GSM_ESCAPE = 0x1b
 
-  CHAR_TABLE = [
+  GSM_TABLE = [
     '@', '£', '$', '¥', 'è', 'é', 'ù', 'ì',
-    'ò', 'Ç',  NL, 'Ø', 'ø', CR , 'Å', 'å',
+    'ò', 'Ç',  nl, 'Ø', 'ø', cr , 'Å', 'å',
     'Δ', '_', 'Φ', 'Γ', 'Λ', 'Ω', 'Π', 'Ψ',
-    'Σ', 'Θ', 'Ξ', " ", 'Æ', 'æ', 'ß', 'É', # 0x1B is actually an escape which we'll encode to a space char
+    'Σ', 'Θ', 'Ξ', nil, 'Æ', 'æ', 'ß', 'É', # 0x1b is the escape
     " ", '!', '"', '#', '¤', '%', '&', "'",
     '(', ')', '*', '+', ',', '-', '.', '/',
     '0', '1', '2', '3', '4', '5', '6', '7',
@@ -31,73 +41,159 @@ module GSMEncoder
     'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
     'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
     'x', 'y', 'z', 'ä', 'ö', 'ñ', 'ü', 'à',
-  ].join # make it string to speedup lookup
-
-  # Extended character table. Characters in this table are accessed by the
-  # 'escape' character in the base table. It is important that none of the
-  # 'inactive' characters ever be matchable with a valid base-table
-  # character as this breaks the encoding loop.
-  EXT_CHAR_TABLE = [
-    0,   0,   0, 0, 0,   0,   0, 0, 0,   'ç', 0, 0, 0,   0,   0,   0,
-    0,   0,   0, 0, '^', 0,   0, 0, 0,   0,   0, 0, 0,   0,   0,   0,
-    0,   0,   0, 0, 0,   0,   0, 0, '{', '}', 0, 0, 0,   0,   0,   BS,
-    0,   0,   0, 0, 0,   0,   0, 0, 0,   0,   0, 0, '[', '~', ']', 0,
-    '|', 'Á', 0, 0, 0,   0,   0, 0, 0,   'Í', 0, 0, 0,   0,   0,   'Ó',
-    0,   0,   0, 0, 0,   'Ú', 0, 0, 0,   0,   0, 0, 0,   0,   0,   0,
-    0,   'á', 0, 0, 0,   '€', 0, 0, 0,   'í', 0, 0, 0,   0,   0,   'ó',
-    0,   0,   0, 0, 0,   'ú', 0, 0, 0,   0,   0, 0, 0,   0,   0,   0,
-  ]
-
-  REGEX = /\A[ -_a-~#{Regexp.escape(CHAR_TABLE + EXT_CHAR_TABLE.select {|c| c != 0}.join)}]*\Z/
-
-  # Verifies that this charset can represent every character in the Ruby
-  # String.
-  # @param str The String to verfiy
-  # @return True if the charset can represent every character in the Ruby
-  #   String, otherwise false.
-  def can_encode?(str)
-    !str || !!(REGEX =~ str)
-  end
+  ].freeze
 
-  def encode(str, replace_char=nil)
-    return nil if !str
+  # GSM 03.38 default extension table. This is the only extension table a
+  # handset (or SMS provider) uses unless the message explicitly selects a
+  # national language shift table in its header.
+  STANDARD_EXT_TABLE = [
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,  ff, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, '^', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, '{', '}', nil, nil, nil, nil, nil, bs,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, '[', '~', ']', nil,
+    '|', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, '€', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+  ].freeze
 
-    replace_char = DEFAULT_REPLACE_CHAR if !replace_char || !can_encode?(replace_char)
+  # Spanish national language single shift table (3GPP TS 23.038 A.2.2).
+  # Only decoded correctly when the message header selects the Spanish shift.
+  SPANISH_EXT_TABLE = [
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, 'ç',  ff, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, '^', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, '{', '}', nil, nil, nil, nil, nil, bs,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, '[', '~', ']', nil,
+    '|', 'Á', nil, nil, nil, nil, nil, nil, nil, 'Í', nil, nil, nil, nil, nil, 'Ó',
+    nil, nil, nil, nil, nil, 'Ú', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, 'á', nil, nil, nil, '€', nil, nil, nil, 'í', nil, nil, nil, nil, nil, 'ó',
+    nil, nil, nil, nil, nil, 'ú', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+  ].freeze
 
-    buffer = ''.encode('binary')
+  # kept for backwards compatibility, this was the only extension table before
+  GSM_EXT_TABLE = SPANISH_EXT_TABLE
+
+  UTF8_ARRAY_SIZE = 128
+
+  # UTF8 to GSM lookups for one extension table
+  Charset = Struct.new(:ext_table, :utf8_array, :utf8_hash, :regexp)
+
+  def self.build_charset(ext_table)
+    unless ext_table.is_a?(Array) && ext_table.size <= UTF8_ARRAY_SIZE
+      raise ArgumentError, "extension table must be an Array of at most #{UTF8_ARRAY_SIZE} entries"
+    end
 
-    begin
-      str.each_char do |c|
-        if index = CHAR_TABLE.rindex(c)
-          buffer << index
-        elsif index = EXT_CHAR_TABLE.index(c)
-          buffer << EXTENDED_ESCAPE
-          buffer << index
+    utf8_array = []
+    utf8_hash = {}
+    [
+      [ GSM_TABLE, nil ],
+      [ ext_table, GSM_ESCAPE ]
+    ].each do |table, prefix|
+      table.each.with_index do |char, index|
+        next if char.nil?
+        # build GSM value
+        if prefix
+          value = +''.encode('binary')
+          value << prefix
+          value << index
         else
-          buffer << replace_char
+          value = index
+        end
+        # store GSM value
+        key = char.codepoints.first
+        if key < UTF8_ARRAY_SIZE
+          utf8_array[key] = value
+        else
+          utf8_hash[key] = value
         end
       end
-    rescue
-      # TODO: ?
     end
-    buffer
+    utf8_array.each(&:freeze)
+    utf8_hash.each_value(&:freeze)
+
+    regexp = /\A[#{Regexp.escape((GSM_TABLE + ext_table).compact.join)}]*\Z/
+
+    Charset.new(ext_table.dup.freeze, utf8_array.freeze, utf8_hash.freeze, regexp).freeze
+  end
+
+  @charsets = {
+    standard: build_charset(STANDARD_EXT_TABLE),
+    spanish: build_charset(SPANISH_EXT_TABLE)
+  }
+
+  # kept for backwards compatibility
+  UTF8_ARRAY = @charsets[:spanish].utf8_array
+  UTF8_HASH = @charsets[:spanish].utf8_hash
+  REGEXP = @charsets[:spanish].regexp
+
+  @extension_table = :spanish
+
+  class << self
+    # The extension table used when a call doesn't pass one (:spanish by default)
+    attr_reader :extension_table
+
+    def extension_table=(name)
+      charset(name) # validate
+      @extension_table = name
+    end
+
+    # Register a custom extension table (an Array indexed by GSM code, nil for unused codes)
+    #
+    #   GSMEncoder.register_extension_table(:mine, table)
+    #   GSMEncoder.extension_table = :mine
+    def register_extension_table(name, table)
+      @charsets = @charsets.merge(name.to_sym => build_charset(table)).freeze
+    end
+
+    def extension_tables
+      @charsets.keys
+    end
+
+    def charset(name = nil)
+      name ||= @extension_table
+      @charsets.fetch(name.to_sym) do
+        raise ArgumentError, "unknown extension table #{name.inspect}, expected one of #{@charsets.keys.inspect}"
+      end
+    end
   end
 
-  def decode bstring
-    return nil if !bstring
+  # Verifies that the given string can be encoded in GSM 03.38
+  def can_encode?(str, extension_table: nil)
+    !str || !!(GSMEncoder.charset(extension_table).regexp =~ str)
+  end
 
-    buffer = ''.encode('utf-8')
+  # Encode given UTF-8 string to GSM 03.38
+  def encode(string, replacement = "?", extension_table: nil)
+    return nil if string.nil?
+    charset = GSMEncoder.charset(extension_table)
+    utf8_array = charset.utf8_array
+    utf8_hash = charset.utf8_hash
+    replacement = replacement == "?" ? 63 : encode(replacement, extension_table: extension_table)
+    buffer = String.new(encoding: "binary")
+    string.each_codepoint do |codepoint|
+      if codepoint < UTF8_ARRAY_SIZE
+        buffer << (utf8_array[codepoint] || replacement)
+      else
+        buffer << (utf8_hash[codepoint] || replacement)
+      end
+    end
+    buffer
+  end
 
-    table = CHAR_TABLE
-    bstring.bytes.each do |c|
-      code = c & 0x000000ff
-      if code == EXTENDED_ESCAPE
-        # take next char from extension table
-        table = EXT_CHAR_TABLE
+  # Encode given GSM 03.38 string to UTF-8
+  def decode(string, replacement = "?", extension_table: nil)
+    return nil if string.nil?
+    ext_table = GSMEncoder.charset(extension_table).ext_table
+    buffer = +""
+    escaped = false
+    string.each_byte do |c|
+      if c == GSM_ESCAPE
+        escaped = true
+      elsif escaped
+        buffer << (ext_table[c] || replacement)
+        escaped = false
       else
-        buffer << (code >= table.length ? '?' : table[code])
-        # go back to the default table
-        table = CHAR_TABLE
+        buffer << (GSM_TABLE[c] || replacement)
       end
     end
     buffer

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gsm_encoder
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Yury Korolev
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-03-18 00:00:00.000000000 Z
+date: 2026-11-12 00:00:00.000000000 Z
 dependencies: []
 description: GSMEncoder encodes and decodes Ruby Strings to and from the SMS default
   alphabet. It also supports the default extension table. The default alphabet and
@@ -41,8 +41,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: ruby GSM 03.38 encoder/decoder