homographic_spoofing 0.1.1 → 0.2.0

data/lib/homographic_spoofing/detector/rule/data/digits.csv

@@ -388,6 +388,16 @@
 𐴷,hanifi_rohingya_digit
 𐴸,hanifi_rohingya_digit
 𐴹,hanifi_rohingya_digit
+𐵀,garay_digit
+𐵁,garay_digit
+𐵂,garay_digit
+𐵃,garay_digit
+𐵄,garay_digit
+𐵅,garay_digit
+𐵆,garay_digit
+𐵇,garay_digit
+𐵈,garay_digit
+𐵉,garay_digit
 𑁦,brahmi_digit
 𑁧,brahmi_digit
 𑁨,brahmi_digit
@@ -478,6 +488,26 @@
 𑛇,takri_digit
 𑛈,takri_digit
 𑛉,takri_digit
+𑛐,myanmar_pao_digit
+𑛑,myanmar_pao_digit
+𑛒,myanmar_pao_digit
+𑛓,myanmar_pao_digit
+𑛔,myanmar_pao_digit
+𑛕,myanmar_pao_digit
+𑛖,myanmar_pao_digit
+𑛗,myanmar_pao_digit
+𑛘,myanmar_pao_digit
+𑛙,myanmar_pao_digit
+𑛚,myanmar_eastern_pwo_karen_digit
+𑛛,myanmar_eastern_pwo_karen_digit
+𑛜,myanmar_eastern_pwo_karen_digit
+𑛝,myanmar_eastern_pwo_karen_digit
+𑛞,myanmar_eastern_pwo_karen_digit
+𑛟,myanmar_eastern_pwo_karen_digit
+𑛠,myanmar_eastern_pwo_karen_digit
+𑛡,myanmar_eastern_pwo_karen_digit
+𑛢,myanmar_eastern_pwo_karen_digit
+𑛣,myanmar_eastern_pwo_karen_digit
 𑜰,ahom_digit
 𑜱,ahom_digit
 𑜲,ahom_digit
@@ -508,6 +538,16 @@
 𑥗,dives_akuru_digit
 𑥘,dives_akuru_digit
 𑥙,dives_akuru_digit
+𑯰,sunuwar_digit
+𑯱,sunuwar_digit
+𑯲,sunuwar_digit
+𑯳,sunuwar_digit
+𑯴,sunuwar_digit
+𑯵,sunuwar_digit
+𑯶,sunuwar_digit
+𑯷,sunuwar_digit
+𑯸,sunuwar_digit
+𑯹,sunuwar_digit
 𑱐,bhaiksuki_digit
 𑱑,bhaiksuki_digit
 𑱒,bhaiksuki_digit
@@ -548,6 +588,16 @@
 𑽗,kawi_digit
 𑽘,kawi_digit
 𑽙,kawi_digit
+𖄰,gurung_khema_digit
+𖄱,gurung_khema_digit
+𖄲,gurung_khema_digit
+𖄳,gurung_khema_digit
+𖄴,gurung_khema_digit
+𖄵,gurung_khema_digit
+𖄶,gurung_khema_digit
+𖄷,gurung_khema_digit
+𖄸,gurung_khema_digit
+𖄹,gurung_khema_digit
 𖩠,mro_digit
 𖩡,mro_digit
 𖩢,mro_digit
@@ -578,6 +628,26 @@
 𖭗,pahawh_hmong_digit
 𖭘,pahawh_hmong_digit
 𖭙,pahawh_hmong_digit
+𖵰,kirat_rai_digit
+𖵱,kirat_rai_digit
+𖵲,kirat_rai_digit
+𖵳,kirat_rai_digit
+𖵴,kirat_rai_digit
+𖵵,kirat_rai_digit
+𖵶,kirat_rai_digit
+𖵷,kirat_rai_digit
+𖵸,kirat_rai_digit
+𖵹,kirat_rai_digit
+𜳰,outlined_digit
+𜳱,outlined_digit
+𜳲,outlined_digit
+𜳳,outlined_digit
+𜳴,outlined_digit
+𜳵,outlined_digit
+𜳶,outlined_digit
+𜳷,outlined_digit
+𜳸,outlined_digit
+𜳹,outlined_digit
 𝟎,mathematical_bold_digit
 𝟏,mathematical_bold_digit
 𝟐,mathematical_bold_digit
@@ -658,6 +728,16 @@
 𞓷,nag_mundari_digit
 𞓸,nag_mundari_digit
 𞓹,nag_mundari_digit
+𞗱,ol_onal_digit
+𞗲,ol_onal_digit
+𞗳,ol_onal_digit
+𞗴,ol_onal_digit
+𞗵,ol_onal_digit
+𞗶,ol_onal_digit
+𞗷,ol_onal_digit
+𞗸,ol_onal_digit
+𞗹,ol_onal_digit
+𞗺,ol_onal_digit
 𞥐,adlam_digit
 𞥑,adlam_digit
 𞥒,adlam_digit

data/lib/homographic_spoofing/detector/rule/disallowed_characters.rb

@@ -1,15 +1,22 @@
 # 3. and 4. of Google Chrome IDN policy See https://util.unicode.org/UnicodeJsps/list-unicodeset.jsp?a=%5B%3AIdentifierStatus%3DAllowed%3A&abb=on&g=&i=
 class HomographicSpoofing::Detector::Rule::DisallowedCharacters < HomographicSpoofing::Detector::Rule::Base
   class << self
-    # See http://kb.mozillazine.org/Network.IDN.blacklist_chars
-    MOZZILLA_DISALLOWED_CHARACTERS = Set[
+    # Characters removed from the allowed set. Combines Mozilla's blocklist
+    # (http://kb.mozillazine.org/Network.IDN.blacklist_chars) with the
+    # removals in Chromium's IDNSpoofChecker::SetAllowedUnicodeSet
+    # (components/url_formatter/spoof_checks/idn_spoof_checker.cc).
+    DISALLOWED_CHARACTERS = Set[
       "\u0020", # Space
       "\u00a0", # No-break space
       "\u00bc", # Vulgar fraction one quarter
       "\u00bd", # Vulgar fraction one half
       "\u00be", # Vulgar fraction three quarters
+      "\u0138", # Latin small letter kra (Chromium)
       "\u01c3", # Latin letter retroflex click
+      "\u02bb", # Modifier letter turned comma (Chromium)
+      "\u02bc", # Modifier letter apostrophe (Chromium)
       "\u02d0", # Modifier letter triangular colon
+      "\u02ec", # Modifier letter voicing (Chromium)
       "\u0337", # Combining short solidus overlay
       "\u0338", # Combining long solidus overlay
       "\u0589", # Armenian full stop
@@ -124,11 +131,11 @@ class HomographicSpoofing::Detector::Rule::DisallowedCharacters < HomographicSpo
     ]
 
     def allowed_chars_set
-      @@allowed_chars_set ||= (read_allowed_idn_chars.chars.to_set - MOZZILLA_DISALLOWED_CHARACTERS)
+      @@allowed_chars_set ||= (read_allowed_idn_chars.chars.to_set - DISALLOWED_CHARACTERS)
     end
 
     private
-      # Built with script/development/generate_allowed_idn_characters.rb
+      # Built with bin/generate_allowed_idn_characters
       def read_allowed_idn_chars
         File.read("#{__dir__}/data/allowed_idn_characters.txt")
       end

data/lib/homographic_spoofing/detector/rule/idn/base.rb

@@ -1,3 +1,3 @@
 class HomographicSpoofing::Detector::Rule::Idn::Base < HomographicSpoofing::Detector::Rule::Base
-  delegate :tld, to: :@context
+  delegate :tld, :tld_allowances, to: :@context
 end

data/lib/homographic_spoofing/detector/rule/idn/context.rb

@@ -1,8 +1,14 @@
 class HomographicSpoofing::Detector::Rule::Idn::Context < HomographicSpoofing::Detector::Rule::Context
-  attr_reader :tld
+  attr_reader :tld, :tld_allowances
 
-  def initialize(label:, tld:)
+  # `tld_allowances` is shared by every label of one domain, so a decision that
+  # depends only on the TLD is made once per domain rather than once per label.
+  # Under a wildcard suffix such as "*.mm" the TLD carries a label the sender
+  # chose, and scanning it again for every label would cost the label count
+  # times its length.
+  def initialize(label:, tld:, occurrence: 0, tld_allowances: {})
     @tld = tld
-    super(label:)
+    @tld_allowances = tld_allowances
+    super(label:, occurrence:)
   end
 end

data/lib/homographic_spoofing/detector/rule/idn/dangerous_pattern.rb

@@ -16,6 +16,13 @@ class HomographicSpoofing::Detector::Rule::Idn::DangerousPattern < HomographicSp
     [\u30ce\u30f3\u30bd\u30be\u4e36\u4e40\u4e41\u4e3f]
     [^\p{kana}\p{hira}\p{hani}]/x,
 
+    /# Rules see one label at a time, so a label edge counts as a non-Japanese
+    # side too: the dot the renderer draws there is not Japanese either. Beside
+    # an edge, "ー" (U+30FC, Common to Unicode's Script property but Japanese in
+    # use) counts as Japanese, so words like "ノート" and "コーン" stay clear.
+    \A[\u30ce\u30f3\u30bd\u30be\u4e36\u4e40\u4e41\u4e3f](?:[^\p{kana}\p{hira}\p{hani}\u30fc]|\z)|
+    [^\p{kana}\p{hira}\p{hani}\u30fc][\u30ce\u30f3\u30bd\u30be\u4e36\u4e40\u4e41\u4e3f]\z/x,
+
       /# Disallow three Hiragana letters (U+307[8-A]) or Katakana letters
       # (U+30D[8-A]) that look exactly like each other when they're used
       # in a label otherwise entirely in Katakana or Hiragana.
@@ -49,6 +56,14 @@ class HomographicSpoofing::Detector::Rule::Idn::DangerousPattern < HomographicSp
       [\u4e00\u3127\u4e28\u4e5b\u4e03\u4e05\u5341\u3007\u3112\u311a\u311f\u3128\u3129\u3108\u31ba\u31b3\u5de5\u31b2\u8ba0\u4e01]
     [^\p{kana}\p{hira}\p{hani}\p{bopo}]/x,
 
+      /# Rules see one label at a time, so a label that is only one of these
+      # ideographs has the dots the renderer draws on both sides, and neither
+      # is CJK: "a.一.example.com" reads as "a.-.example.com". A longer label
+      # with one at its edge is already caught by the two patterns above when
+      # its other neighbour is non-CJK, and stays safe when it is CJK, as in
+      # "工行" or "一号店".
+      \A[一ㄧ丨乛七丅十〇ㄒㄚㄟㄨㄩㄈㆺㆳ工ㆲ讠丁]\z/x,
+
       /# Disallow combining diacritical mark (U+0300-U+0339) after a
       # non-LGC character. Other combining diacritical marks are not in
       # the allowed character set.

data/lib/homographic_spoofing/detector/rule/idn/invisible_characters.rb

@@ -9,6 +9,6 @@ class HomographicSpoofing::Detector::Rule::Idn::InvisibleCharacters < Homographi
       /\u0e48{2,}/,   # Thai tone repeated
       /\u0301{2,}/,   # accute accent repeated
       /\u00e1\u0301/, # 'a' with acuted accent + another acute accent
-      /^\u0300/,      # Combining mark at the beginning
+      /\A\p{M}/,      # Combining mark at the beginning of a label
     )
 end

data/lib/homographic_spoofing/detector/rule/idn/script_confusable.rb

@@ -46,11 +46,23 @@ class HomographicSpoofing::Detector::Rule::Idn::ScriptConfusable < HomographicSp
       [ /\p{Mymr}/, /[ခဂငထပဝ၀၂ၔၜ\u1090\u1091\u1095\u1096\u1097]/, /[a-z]+\.mm/ ],
       # # Thai
       [ /\p{Thai}/, /[ทนบพรหเแ๐ดลปฟม]/, /th/ ]
-    ].map { Confusable.new(*_1) }
+    ]
+    # Anchor each allowed-TLD pattern so it only matches whole trailing labels
+    # of the TLD, never a substring of one. Unanchored, `/ru/` matched `.guru`,
+    # `/su/` matched `.surf`, `/by/` matched `.baby`/`.rugby`, etc., silently
+    # disabling single-script confusable detection on those TLDs. The
+    # `(?:\A|\.)…\z` boundary keeps multi-label public suffixes working —
+    # `co.th` still matches `/th/`, `com.mm` still matches `/[a-z]+\.mm/`.
+    .map do |script, latin_lookalike, allowed_tlds|
+      anchored_tlds = allowed_tlds && /(?:\A|\.)(?:#{allowed_tlds.source})\z/
+      Confusable.new script, latin_lookalike, anchored_tlds
+    end
 
     def is_script_confusable_allowed_for_tld?(confusable)
-      tld_contains_any_letter_from_script?(confusable.script) ||
-        confusable.allowed_tlds&.match?(tld)
+      tld_allowances.fetch(confusable) do
+        tld_allowances[confusable] = tld_contains_any_letter_from_script?(confusable.script) ||
+          confusable.allowed_tlds&.match?(tld) || false
+      end
     end
 
     def tld_contains_any_letter_from_script?(script)

data/lib/homographic_spoofing/detector/rule/mixed_digits.rb

@@ -23,7 +23,7 @@ class HomographicSpoofing::Detector::Rule::MixedDigits < HomographicSpoofing::De
       end
     end
 
-    # Built with script/development/generate_digits_characters.rb
+    # Built with bin/generate_digits_characters
     def read_digits
       File.read("#{__dir__}/data/digits.csv")
     end

data/lib/homographic_spoofing/sanitizer/base.rb

@@ -10,19 +10,153 @@ class HomographicSpoofing::Sanitizer::Base
   end
 
   def sanitize
-    result = field.dup
-    detector_class.new(field).detections.each do |detection|
-      log(detection.reason, detection.label)
-      result = punycode(result, detection.label)
-    end
-    result
+    detections = detector_class.new(field).detections
+    detections.each { |detection| log(detection.reason, detection.label) }
+    apply(field, detections)
   end
 
   private
     attr_reader :field
 
-    def punycode(source, label)
-      source.gsub(label, Dnsruby::Name.punycode(label))
+    # Punycode each offending label only within the component it came from. A
+    # spanned detection carries the [offset, length] of its component, so a
+    # domain-label spoof is confined to the domain and never rewrites a benign
+    # local part that merely equals the same string — this is what keeps
+    # "з@мир.з.example.com" punycoding the domain label "з" while leaving the "з"
+    # mailbox intact. Spans are disjoint or nested (a display name read from a
+    # comment the host's token carries), so they form a tree: each span's text is
+    # rebuilt with its inner spans already rewritten, then rewritten itself, in
+    # one pass over the field. A detection with no span (a bare IDN or quoted
+    # string) spans the whole field and is applied last, over what remains.
+    def apply(field, detections)
+      return field.dup if detections.empty?
+
+      whole, spanned = detections.partition { |detection| detection.span.nil? }
+      edits = spanned.group_by(&:span).map do |(offset, length), group|
+        Edit.new(offset, offset + length, group.map(&:label).uniq, [])
+      end
+      chars = field.chars
+      replace_labels(splice(chars, 0, chars.length, nest(edits)), whole.map(&:label).uniq)
+    end
+
+    Edit = Struct.new(:start, :finish, :labels, :inner)
+
+    # Arrange spans into a tree, outer before inner, in one pass over them sorted
+    # by start. A span that overlaps an earlier sibling without nesting in it is
+    # trimmed to start where that sibling ends.
+    def nest(edits)
+      outermost, open = [], []
+      edits.sort_by { |edit| [ edit.start, -edit.finish ] }.each do |edit|
+        open.pop while open.any? && open.last.finish < edit.finish
+        siblings = open.empty? ? outermost : open.last.inner
+        edit.start = [ edit.start, siblings.last.finish ].max if siblings.any?
+        next if edit.start > edit.finish
+
+        siblings << edit
+        open << edit
+      end
+      outermost
+    end
+
+    def splice(chars, start, finish, edits)
+      text, at = +"", start
+      edits.each do |edit|
+        text << chars[at...edit.start].join << replace_labels(splice(chars, edit.start, edit.finish, edit.inner), edit.labels)
+        at = edit.finish
+      end
+      text << chars[at...finish].join
+    end
+
+    # Replace each label where it is a whole "."-delimited component of the
+    # region, so an offending label is never rewritten as a substring of a benign
+    # sibling (the Cyrillic digit-look-alike "з" that also sits inside
+    # "магазин"), and every matching component is punycoded from its own
+    # spelling — case-exact, so a spoof repeated in different casing is handled
+    # per occurrence. A component is compared with its CFWS (comments and
+    # surrounding whitespace, which the parser drops from the label it reports)
+    # set aside, and only the label itself is rewritten, so "з(comment)" is
+    # punycoded at the label and a comment beside it never sends the replacement
+    # into a benign sibling. A label that is only a substring of a component (a
+    # display name carrying spaces) has no whole component to match and falls
+    # back to exact substring replacement.
+    #
+    # A label that keeps whitespace inside it, as PublicSuffix reports "а " in
+    # "А .com", matches its component as written.
+    #
+    # The region is scanned once for all labels, so sanitizing stays linear in
+    # its length however many labels were detected.
+    def replace_labels(region, labels)
+      return region if labels.empty?
+
+      components = label_components(region)
+      present = components.to_set(&:bare) | components.map(&:text)
+      whole, partial = labels.partition { |label| present.include?(label) }
+      whole = whole.to_set
+
+      result = components.map do |component|
+        if whole.include?(component.text) && component.bare
+          Dnsruby::Name.punycode(component.text)
+        elsif whole.include?(component.bare)
+          component.punycoded
+        else
+          component.text
+        end
+      end.join
+      partial.inject(result) { |text, label| text.gsub(label) { Dnsruby::Name.punycode(label) } }
+    end
+
+    # One "."-delimited component of a region: its text, the label it carries
+    # with CFWS set aside (`bare`), and where that label starts in the text, past
+    # any leading CFWS that may repeat it. Separators are components too, with no
+    # label.
+    Component = Struct.new(:text, :bare, :start) do
+      def punycoded
+        if text[start, bare.length] == bare
+          text[0, start] + Dnsruby::Name.punycode(bare) + text[(start + bare.length)..]
+        else
+          text
+        end
+      end
+    end
+
+    # Split a region into label components on each "." outside comments and
+    # quoted strings, in a single pass. Comments nest and honour backslash
+    # escapes, as RFC 5322 has them, so a comment's own dots or parentheses
+    # never split or end a label early.
+    def label_components(region)
+      components = []
+      text, bare, start = +"", +"", nil
+      depth, quoted, escaped = 0, false, false
+
+      region.each_char do |char|
+        commented = depth > 0
+        if escaped
+          escaped = false
+        elsif char == "\\" && (commented || quoted)
+          escaped = true
+        elsif commented
+          depth += 1 if char == "("
+          depth -= 1 if char == ")"
+        elsif quoted
+          quoted = false if char == '"'
+        elsif char == "("
+          depth, commented = 1, true
+        elsif char == '"'
+          quoted = true
+        elsif char == "."
+          components << Component.new(text, bare.strip, start || 0) << Component.new(char, nil, 0)
+          text, bare, start = +"", +"", nil
+          next
+        end
+
+        unless commented
+          start ||= text.length unless char.match?(/\s/)
+          bare << char
+        end
+        text << char
+      end
+
+      components << Component.new(text, bare.strip, start || 0)
     end
 
     def detector_class

data/lib/homographic_spoofing/sanitizer/idn.rb

@@ -7,4 +7,34 @@ class HomographicSpoofing::Sanitizer::Idn < HomographicSpoofing::Sanitizer::Base
     def spoofing_type
       "EmailIDN"
     end
+
+    # A bare domain has no comments or quoted strings: the detector splits it on
+    # every ".", so split it the same way. Honouring "(" or '"' here would let an
+    # unmatched one fold the rest of the domain into one component, and every
+    # detected label would then fall back to a substitution over the whole field.
+    #
+    # The detector reports labels as PublicSuffix does: with the domain's own
+    # leading and trailing padding set aside, but any whitespace inside the
+    # domain kept on its label ("А " in " А .А .com"). That padding is set apart
+    # as components with no label, so each label component reads as its label
+    # was reported.
+    def label_components(region)
+      core = region.strip
+      leading = core.empty? ? region : region[0, region.index(core)]
+      trailing = region[(leading.length + core.length)..]
+
+      [ padding(leading), *core.split(/(\.)/, -1).map { |text| component(text) }, padding(trailing) ].compact
+    end
+
+    def component(text)
+      if text == "."
+        Component.new(text, nil, 0)
+      else
+        Component.new(text, text.strip, text.index(/\S/) || 0)
+      end
+    end
+
+    def padding(text)
+      Component.new(text, nil, 0) unless text.empty?
+    end
 end

data/lib/homographic_spoofing/version.rb

@@ -1,3 +1,3 @@
 module HomographicSpoofing
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: homographic_spoofing
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jacopo Beschi
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-20 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: zeitwerk
@@ -94,13 +93,27 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 email:
 - jacopo@37signals.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
 - MIT-LICENSE
 - README.md
 - lib/homographic_spoofing.rb
@@ -145,7 +158,9 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/basecamp/homographic_spoofing
   source_code_uri: https://github.com/basecamp/homographic_spoofing
-post_install_message:
+  changelog_uri: https://github.com/basecamp/homographic_spoofing/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/basecamp/homographic_spoofing/issues
+  rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
 - lib
@@ -160,8 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
-signing_key:
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A toolkit to both detect and sanitize homographic spoofing attacks in URLs
   and Email addresses