gsm_encoder 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be392aec01587b907e1e0454bec610cc068359a88d614cb31850d5208503cf9d
-  data.tar.gz: 13b2b3a007cae53be426a8da7413676e020554cccf62ef838b87e4266f71d368
+  metadata.gz: 328cd3d4611243a39035910afc138fda9bdcc23620edc2c6fa83eb4bf9bfa4dc
+  data.tar.gz: 450843ca2309b3070ed92835b5e81620c7bf38467c8e435017e5692c3e8cce7c
 SHA512:
-  metadata.gz: fc9ace5dc2f040ad1fc4ed994dc43c74c3db7b513cdf9d6a1cf3491eb794635dc64c88e6c5ad2c3fe73cc1b9d4d65b9202d388cfc9ee32f46b0bf4c6eac2301b
-  data.tar.gz: 4edfb38d92c90f16963bea9525e47f082af4a320badd64b79b1e382f56abc8c291a3cd3fa0c8187fcfdff9cc8cb5ce1ad5724928c214c7e1769f5cca12244b10
+  metadata.gz: 98bd3e1a50f2e05b957cff8c041666e4647d8e9c0ef28ad795ef610fa5423e9978d9015b318a9b116e6dd4b65deea03b0ed8298c7c04eba50452ec59ce43aece
+  data.tar.gz: e2b7a8daacf2c746946fc772b0eb22aa344de19e811322c637c24d810cb50c455db399b2b1dc10c55371c533d2f46726700aac148701d55faa0d08eb76421eea

data/README.md

@@ -29,6 +29,35 @@ __NOTE: ruby >= 1.9.2 is required__
     # replaces unsupported chars with a provided value ' '
     GSMEncoder.encode('`', ' ') # => ' '
 
+## Extension tables
+
+Characters like `{ } [ ] ~ \ | € ^` are sent as an escape plus a code from an
+extension table. Two tables are built in:
+
+- `:spanish` (the default) - the Spanish national language single shift table.
+  It adds `ç Á Í Ó Ú á í ó ú` on top of the standard table.
+- `:standard` - the GSM 03.38 default extension table.
+
+The Spanish characters are only valid GSM 7-bit when the message header selects
+the Spanish shift table. Most SMS providers (Twilio, Telnyx, Plivo, ...) don't,
+so a message containing them is sent as UCS-2 instead. That cuts a segment from
+160 characters to 70. Use `:standard` if you rely on `can_encode?` or
+`encode` to keep messages in GSM 7-bit:
+
+    # for every call
+    GSMEncoder.extension_table = :standard
+
+    # or for one call
+    GSMEncoder.can_encode?('á', extension_table: :standard) # => false
+    GSMEncoder.encode('á', extension_table: :standard) # => '?'
+    GSMEncoder.decode(bytes, extension_table: :standard)
+
+You can also register your own table, an Array indexed by GSM code with `nil`
+for unused codes:
+
+    GSMEncoder.register_extension_table(:mine, table)
+    GSMEncoder.extension_table = :mine
+
 ## Updates
 
 ### 0.1.1
@@ -55,3 +84,10 @@ Even faster. About 4 times. Many thanks to @dlarrabee
 ### 0.1.7
 
 Even faster! Many thanks to @ThomasSevestre
+
+### 0.1.8
+
+- Configurable extension table (`:standard` or `:spanish`), defaults to `:spanish`
+- Adds form feed to the extension tables
+- `decode` returns the replacement for unknown codes instead of raising `TypeError`
+

data/lib/gsm_encoder.rb

@@ -4,11 +4,21 @@
 
 #
 # This class encodes and decodes Ruby Strings to and from the SMS default
-# alphabet. It also supports the default extension table. The default alphabet
-# and it's extension table is defined in GSM 03.38.
+# alphabet. It also supports an extension table. The default alphabet and
+# its extension tables are defined in GSM 03.38 (3GPP TS 23.038).
+#
+# Which extension table is used can be configured:
+#
+#   GSMEncoder.extension_table = :standard # GSM 03.38 default extension table only
+#   GSMEncoder.extension_table = :spanish  # Spanish single shift table (the default)
+#
+# or per call:
+#
+#   GSMEncoder.encode("¿Qué?", extension_table: :standard)
 module GSMEncoder
   # GSM to UTF8 tables
   nl = 10.chr
+  ff = 12.chr
   cr = 13.chr
   bs = 92.chr
 
@@ -33,8 +43,24 @@ module GSMEncoder
     'x', 'y', 'z', 'ä', 'ö', 'ñ', 'ü', 'à',
   ].freeze
 
-  GSM_EXT_TABLE = [
-    nil, nil, nil, nil, nil, nil, nil, nil, nil, 'ç', nil, nil, nil, nil, nil, nil,
+  # GSM 03.38 default extension table. This is the only extension table a
+  # handset (or SMS provider) uses unless the message explicitly selects a
+  # national language shift table in its header.
+  STANDARD_EXT_TABLE = [
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,  ff, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, '^', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, '{', '}', nil, nil, nil, nil, nil, bs,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, '[', '~', ']', nil,
+    '|', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, '€', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
+  ].freeze
+
+  # Spanish national language single shift table (3GPP TS 23.038 A.2.2).
+  # Only decoded correctly when the message header selects the Spanish shift.
+  SPANISH_EXT_TABLE = [
+    nil, nil, nil, nil, nil, nil, nil, nil, nil, 'ç',  ff, nil, nil, nil, nil, nil,
     nil, nil, nil, nil, '^', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
     nil, nil, nil, nil, nil, nil, nil, nil, '{', '}', nil, nil, nil, nil, nil, bs,
     nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, '[', '~', ']', nil,
@@ -44,73 +70,130 @@ module GSMEncoder
     nil, nil, nil, nil, nil, 'ú', nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,
   ].freeze
 
-  # build UTF8 to GSM tables
+  # kept for backwards compatibility, this was the only extension table before
+  GSM_EXT_TABLE = SPANISH_EXT_TABLE
+
   UTF8_ARRAY_SIZE = 128
-  UTF8_ARRAY = []
-  UTF8_HASH = {}
-  [
-    [ GSM_TABLE, nil ],
-    [ GSM_EXT_TABLE, GSM_ESCAPE ]
-  ].each do |table, prefix|
-    table.each.with_index do |char, index|
-      next if char.nil?
-      # build GSM value
-      if prefix
-        value = +''.encode('binary')
-        value << prefix
-        value << index
-      else
-        value = index
-      end
-      # store GSM value
-      key = char.codepoints.first
-      if key < UTF8_ARRAY_SIZE
-        UTF8_ARRAY[key] = value
-      else
-        UTF8_HASH[key] = value
+
+  # UTF8 to GSM lookups for one extension table
+  Charset = Struct.new(:ext_table, :utf8_array, :utf8_hash, :regexp)
+
+  def self.build_charset(ext_table)
+    unless ext_table.is_a?(Array) && ext_table.size <= UTF8_ARRAY_SIZE
+      raise ArgumentError, "extension table must be an Array of at most #{UTF8_ARRAY_SIZE} entries"
+    end
+
+    utf8_array = []
+    utf8_hash = {}
+    [
+      [ GSM_TABLE, nil ],
+      [ ext_table, GSM_ESCAPE ]
+    ].each do |table, prefix|
+      table.each.with_index do |char, index|
+        next if char.nil?
+        # build GSM value
+        if prefix
+          value = +''.encode('binary')
+          value << prefix
+          value << index
+        else
+          value = index
+        end
+        # store GSM value
+        key = char.codepoints.first
+        if key < UTF8_ARRAY_SIZE
+          utf8_array[key] = value
+        else
+          utf8_hash[key] = value
+        end
       end
     end
+    utf8_array.each(&:freeze)
+    utf8_hash.each_value(&:freeze)
+
+    regexp = /\A[#{Regexp.escape((GSM_TABLE + ext_table).compact.join)}]*\Z/
+
+    Charset.new(ext_table.dup.freeze, utf8_array.freeze, utf8_hash.freeze, regexp).freeze
   end
-  UTF8_ARRAY.freeze
-  UTF8_ARRAY.each(&:freeze)
-  UTF8_HASH.freeze
-  UTF8_HASH.each_value(&:freeze)
 
-  REGEXP = /\A[#{Regexp.escape((GSM_TABLE + GSM_EXT_TABLE).compact.join)}]*\Z/
+  @charsets = {
+    standard: build_charset(STANDARD_EXT_TABLE),
+    spanish: build_charset(SPANISH_EXT_TABLE)
+  }
+
+  # kept for backwards compatibility
+  UTF8_ARRAY = @charsets[:spanish].utf8_array
+  UTF8_HASH = @charsets[:spanish].utf8_hash
+  REGEXP = @charsets[:spanish].regexp
+
+  @extension_table = :spanish
+
+  class << self
+    # The extension table used when a call doesn't pass one (:spanish by default)
+    attr_reader :extension_table
+
+    def extension_table=(name)
+      charset(name) # validate
+      @extension_table = name
+    end
+
+    # Register a custom extension table (an Array indexed by GSM code, nil for unused codes)
+    #
+    #   GSMEncoder.register_extension_table(:mine, table)
+    #   GSMEncoder.extension_table = :mine
+    def register_extension_table(name, table)
+      @charsets = @charsets.merge(name.to_sym => build_charset(table)).freeze
+    end
+
+    def extension_tables
+      @charsets.keys
+    end
+
+    def charset(name = nil)
+      name ||= @extension_table
+      @charsets.fetch(name.to_sym) do
+        raise ArgumentError, "unknown extension table #{name.inspect}, expected one of #{@charsets.keys.inspect}"
+      end
+    end
+  end
 
   # Verifies that the given string can be encoded in GSM 03.38
-  def can_encode?(str)
-    !str || !!(REGEXP =~ str)
+  def can_encode?(str, extension_table: nil)
+    !str || !!(GSMEncoder.charset(extension_table).regexp =~ str)
   end
 
   # Encode given UTF-8 string to GSM 03.38
-  def encode(string, replacement = "?")
+  def encode(string, replacement = "?", extension_table: nil)
     return nil if string.nil?
-    replacement = replacement == "?" ? 63 : encode(replacement)
+    charset = GSMEncoder.charset(extension_table)
+    utf8_array = charset.utf8_array
+    utf8_hash = charset.utf8_hash
+    replacement = replacement == "?" ? 63 : encode(replacement, extension_table: extension_table)
     buffer = String.new(encoding: "binary")
     string.each_codepoint do |codepoint|
       if codepoint < UTF8_ARRAY_SIZE
-        buffer << (UTF8_ARRAY[codepoint] || replacement)
+        buffer << (utf8_array[codepoint] || replacement)
       else
-        buffer << (UTF8_HASH[codepoint] || replacement)
+        buffer << (utf8_hash[codepoint] || replacement)
       end
     end
     buffer
   end
 
   # Encode given GSM 03.38 string to UTF-8
-  def decode(string, replacement = "?")
+  def decode(string, replacement = "?", extension_table: nil)
     return nil if string.nil?
+    ext_table = GSMEncoder.charset(extension_table).ext_table
     buffer = +""
     escaped = false
     string.each_byte do |c|
       if c == GSM_ESCAPE
         escaped = true
       elsif escaped
-        buffer << GSM_EXT_TABLE[c] || replacement
+        buffer << (ext_table[c] || replacement)
         escaped = false
       else
-        buffer << GSM_TABLE[c] || replacement
+        buffer << (GSM_TABLE[c] || replacement)
       end
     end
     buffer

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gsm_encoder
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Yury Korolev
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-09-13 00:00:00.000000000 Z
+date: 2026-11-12 00:00:00.000000000 Z
 dependencies: []
 description: GSMEncoder encodes and decodes Ruby Strings to and from the SMS default
   alphabet. It also supports the default extension table. The default alphabet and