string_tools 0.2.0 → 0.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MzJkYTUyMTM2ODcwZGMzYjg3NWQ1ZTE4MTQ2YzEyYmY1MjdhMzg1MA==
+    ZDQzNWJhM2NkOTkyMzVlYWQ4N2Y2YWM4OTEwZjQ3MTZjMjMyYzc0Ng==
   data.tar.gz: !binary |-
-    ODJhNTU1Mzg5YTNkNTNkMjJkNmRlZjk5MzRhN2QzOTNjNjEyYjYwYg==
+    MTZmZTMyYjcwY2NmYjcwMzZlZjg4MTk0NDU0ODAzZDY3NmVjZDQyNQ==
 SHA512:
   metadata.gz: !binary |-
-    YzFjYWZlNzY5M2ZmZGU2MDYyZWQ0ZjJlZmM3NDhiMzE5MzMyYTU5MGJhNmY0
-    YjIzMTI2YzBkNDk0MjdlZjA2MWRjMzU0MWM5NmE4NTE3OGY4N2RhNzYxNmFk
-    NjFlMzNlYWY5YWVlMzIwYzc4NzhjODY0ZTk1ZGI3MmYzMzllYjU=
+    ODUyNWM1ODVkZDYxMzhlZTk5OTlhNjRhOTNjZTI5NTA5OTAzYTZiMjMwMmZk
+    ZmFjYmI3MTEzOTM4NWU0NDkzNTk1NzkxMDNmNGQ4YjgzNjg1ZDRmOGE3YmUx
+    YmJiM2RlMDUwMWVlZjJlMGNhZGYzNjdmNTYwOGZjNTRhMWI4ZTM=
   data.tar.gz: !binary |-
-    NDQyOTRjNTQwZTU5ODViMmRhMWI5NGJhMDBmNWYwYTMxOWRiNDBjYmIwOWE1
-    NTI4Y2U0MGQ3MTEwMWM3MDU0NTlhYjIzZmQyNGE3MGU2OWNmNDU4YTU4ZTFk
-    OTY0MWYxN2VjYjlkMjRmYzI1ODA1ODRiNzhkNzRkM2EwMTI2OTM=
+    YTUzNGZiYmY2MGQ2NmYwNzk4ZDk2OWY4YTVmMThhNDFiZDE1NTEzNTU2ZDlh
+    YjlmNTc4MjA2YmZhNTkxMGExZWMwNWIyMWJhMDFlMDZlZWU2M2NlNWMzYjJj
+    OGQ3ZDQ5MWU2MjM1YWU1MDY0ZmNiNTliZTZjMTk2ZGI2ZDk2ODY=

data/.gitignore

@@ -7,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+/gemfiles

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+
+#### [Current]
+ * 2015-10-26 [62ce841](../../commit/62ce841) - __(Dmitry Bochkarev)__ feature(html): удаление ссылок без хоста по-умолчанию
+ * 2015-10-23 [8fe4384](../../commit/8fe4384) - __(Dmitry Bochkarev)__ fix(sanitizer): нормализация ссылок в юникоде
+ * 2015-10-23 [4017e3d](../../commit/4017e3d) - __(Dmitry Bochkarev)__ fix(html): кириллические ссылки в урлах
+ * 2015-10-23 [e05076f](../../commit/e05076f) - __(Dmitry Bochkarev)__ fix(html): поддержка относительных путей
+ * 2015-10-14 [24bd113](../../commit/24bd113) - __(Dmitry Bochkarev)__ chore: костанта с минимальным размером строки содержащей ссылки
+ * 2015-10-12 [e48da9f](../../commit/e48da9f) - __(Dmitry Bochkarev)__ feature: удаление ссылок из текста
+ * 2015-08-07 [274f820](../../commit/274f820) - __(evseevleo)__ feature(strip_tags): removing open comment tags
+ * 2015-07-20 [94b855d](../../commit/94b855d) - __(Sergey D)__ Release 0.2.0
+ * 2015-07-18 [81cb0f1](../../commit/81cb0f1) - __(Sergey D)__ feat: missing String.natcmp & Colorize methods
+
+#### v0.1.0
+ * 2015-07-15 [29dd2f8](../../commit/29dd2f8) - __(Sergey D)__ feat: Initial commit
+ * 2015-07-15 [569f0d6](../../commit/569f0d6) - __(Artem Napolskih)__ Initial commit

data/Gemfile

@@ -2,3 +2,4 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in string_tools.gemspec
 gemspec
+

data/lib/string_tools/core_ext/string.rb

@@ -17,7 +17,7 @@ class String
   # возвращает строку из которой удалены HTML-теги
   # символы <>&"  остаются без изменения
   def strip_tags
-    ActionController::Base.helpers.strip_tags(self).to_str
+    ActionController::Base.helpers.strip_tags(self).to_str.gsub(/<!--/, '<--')
   end
 
   # '11,3'.to_f

data/lib/string_tools/html.rb

@@ -0,0 +1,97 @@
+# coding: utf-8
+require 'nokogiri'
+require 'addressable/uri'
+require 'simpleidn'
+
+module StringTools
+  module HTML
+    # минимальная длина строки, в которой могут быть ссылки
+    TEXT_WITH_LINKS_MINIMUM_LENGTH = '<a href="'.length
+    HTML_SERIALIZE_OPTIONS = {
+      indent: 0,
+      # сериализуем в xhtml, поскольку при сериализации в html, libxml2 делает чуть больше, чем хотелось бы:
+      # http://stackoverflow.com/questions/24174032/prevent-nokogiri-from-url-encoding-src-attributes
+      save_with: Nokogiri::XML::Node::SaveOptions::AS_XHTML
+    }
+
+    # Public: Удаляет ссылки на неразрешенные домены
+    #
+    # html    - String содержимое потенциально ненужных ссылок
+    # options - Hash
+    #         :whitelist - Array of String разрешенныe домены
+    #
+    # Examples
+    #   html = '<a href="https://www.yandex.ru">yandex</a>'
+    #
+    #   StringTools::HTML.remove_links(html, whitelist: ['google.com'])
+    #   # => 'yandex'
+    #
+    #   StringTools::HTML.remove_links(html, whitelist: ['yandex.ru'])
+    #   # => '<a href="https://www.yandex.ru">yandex</a>'
+    #
+    #   StringTools::HTML.remove_links(html, whitelist: ['www.yandex.ru'])
+    #   # => '<a href="https://www.yandex.ru">yandex</a>'
+    #
+    #   html = '<a href="https://yandex.ru">yandex</a>'
+    #
+    #   StringTools::HTML.remove_links(html, whitelist: ['www.yandex.ru'])
+    #   # => 'yandex'
+    #
+    # Returns String without links to external resources
+    def self.remove_links(html, options = {})
+      return html if html.length < TEXT_WITH_LINKS_MINIMUM_LENGTH
+
+      doc = Nokogiri::HTML::DocumentFragment.parse(html)
+      scrubber = LinksRemoveScrubber.new(options)
+
+      doc.css('a'.freeze).each { |node| scrubber.call node }
+
+      if scrubber.done_changes?
+        doc.children.map { |node| node.serialize HTML_SERIALIZE_OPTIONS }.join
+      else
+        html
+      end
+    end
+
+    class LinksRemoveScrubber
+      def initialize(options)
+        @whitelist = options.fetch(:whitelist)
+        @remove_without_host = options.fetch(:remove_without_host, true)
+        @is_have_done_changes = false
+      end
+
+      def done_changes?
+        @is_have_done_changes
+      end
+
+      def call(node)
+        href = node['href']
+        return if href.blank?
+        uri = Addressable::URI.parse(href).normalize
+        if !uri.host
+          replace_with_content node if @remove_without_host
+        elsif !whitelisted?(SimpleIDN.to_unicode(uri.host))
+          replace_with_content node
+        end
+      end
+
+      def whitelisted?(domain)
+        host_parts = domain.split('.'.freeze)
+        host = host_parts[-1] # com, ru ...
+        (host_parts.length - 2).downto(0) do |i|
+          subdomain = host_parts[i]
+          host = "#{subdomain}.#{host}"
+          return true if @whitelist.include? host
+        end
+        false
+      end
+
+      private
+
+      def replace_with_content(node)
+        node.swap(node.children)
+        @is_have_done_changes = true
+      end
+    end
+  end
+end

data/lib/string_tools/version.rb

@@ -1,3 +1,3 @@
 module StringTools
-  VERSION = '0.2.0'
+  VERSION = '0.3.0'
 end

data/lib/string_tools.rb

@@ -6,6 +6,8 @@ require 'active_support/core_ext/string'
 require 'string_tools/core_ext/string'
 
 module StringTools
+  autoload :HTML, 'string_tools/html'
+
   module CharDet
     # Возвращает true если строка содержит допустимую
     # последовательность байтов для кодировки utf8 и false в обратном случае
@@ -108,11 +110,10 @@ module StringTools
 
   module Sanitizer
     class Base
-
       TAGS_WITH_ATTRIBUTES = {
         'p'     => %w(align style),
         'div'   => %w(align style),
-        'span'   => %w(align style),
+        'span'  => %w(align style),
         'td'    => %w(align width valign colspan rowspan style),
         'th'    => %w(align width valign colspan rowspan style),
         'a'     => %w(href target name style),
@@ -135,15 +136,40 @@ module StringTools
         attributes.merge!(attr)
         elements = attributes.keys | TAGS_WITHOUT_ATTRIBUTES
 
-        Sanitize.fragment(str,
+        Sanitize.fragment(
+          str,
           :attributes => attributes,
           :elements => elements,
           :css => {:properties => Sanitize::Config::RELAXED[:css][:properties]},
           :remove_contents => %w(style javascript),
-          :allow_comments => false
+          :allow_comments => false,
+          :transformers => [LINK_NORMALIZER]
         )
       end
     end
+
+    # приводит ссылки согласно стандарту, не корёжит
+    # http://www.фермаежей.рф => http://www.xn--80ajbaetq5a8a.xn--p1ai
+    class LinkNormalizer
+      def call(env)
+        node = env[:node]
+        case node.name
+        when 'a'.freeze
+          normalize_link node, 'href'.freeze
+        when 'img'.freeze
+          normalize_link node, 'src'.freeze
+        end
+      end
+
+      private
+
+      def normalize_link(node, attr_name)
+        return unless node[attr_name]
+        node[attr_name] = Addressable::URI.parse(node[attr_name]).normalize.to_s
+      end
+    end
+
+    LINK_NORMALIZER = LinkNormalizer.new
   end
 
   module SumInWords

data/string_tools.gemspec

@@ -26,13 +26,15 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'addressable', '~> 2.3.2'
   spec.add_runtime_dependency 'ru_propisju', '~> 2.1.4'
   spec.add_runtime_dependency 'sanitize', '>= 3.1.2'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'simpleidn', '>= 0.0.5'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'rspec', '>= 2.14.0'
   spec.add_development_dependency 'rspec-rails', '>= 2.14.0'
   spec.add_development_dependency 'rspec-given', '~> 3.5'
-  spec.add_development_dependency 'shoulda-matchers'
+  spec.add_development_dependency 'shoulda-matchers', '~> 2.0'
   spec.add_development_dependency 'appraisal', '>= 1.0.2'
   spec.add_development_dependency 'combustion', '>= 0.5.3'
   spec.add_development_dependency 'simplecov', '>= 0.9'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: string_tools
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Sergey D.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-20 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -108,6 +108,34 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: 3.1.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simpleidn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.5
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -182,16 +210,16 @@ dependencies:
   name: shoulda-matchers
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -258,6 +286,7 @@ files:
 - .gitignore
 - .rspec
 - Appraisals
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - Makefile
@@ -266,6 +295,7 @@ files:
 - bin/console
 - lib/string_tools.rb
 - lib/string_tools/core_ext/string.rb
+- lib/string_tools/html.rb
 - lib/string_tools/version.rb
 - string_tools.gemspec
 homepage: https://github.com/abak-press/string_tools
@@ -288,7 +318,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: String Tools