html-pipeline 3.0.0.pre1 → 3.0.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b8171e0970d8cbe50757505138bbea0f812c06583cfcdfd192686a8aff9cb53
-  data.tar.gz: 19a40ef93871f3d2b6411c526d6375a21d494a301d7a4abb2e92b6062de81b1c
+  metadata.gz: 613a099c0290d87932d2bbaf78052d8c472518bb911d82fe651ad78a3cc32ee9
+  data.tar.gz: d03826dc83afb689e87395d3a365a24fd5b3608e3a29b632a79a5928acb84a2b
 SHA512:
-  metadata.gz: 4c160dcf8463cc5009d397529087c9f035cb3ae5e998c12ace03810fd8146b031f5f28829e318456239cd410d41fbf3d1da7d0b0e8097ceb3206733a96484c5f
-  data.tar.gz: 72fa1ec6b3a2097dd0ce6108f55a10ac9b617e5fc607e67818876c5b8ececbc7c2bbd58e53ce530ba39f11f6ad0c292ec956e9e42fbed370213abc5b71bcf257
+  metadata.gz: 35ce289f0ca10929fba016a5dd0d7c0a6bf0d8f8d4390385590c0937190df67605b96b474a477a54feb5192a54fc5b299940d264483700a9c5deac04cff1b9e1
+  data.tar.gz: dbddce89afcb5f0291d28c9b24b4a2e4f25ea847d8960dc06a11f0c6cfd6cf2feb169f406fd69ca5f25a3eabc4338ce0cf7f2b94fa2147c682135bb6dee70728

data/CHANGELOG.md

@@ -1,5 +1,43 @@
 # Changelog
 
+## [Unreleased](https://github.com/gjtorikian/html-pipeline/tree/HEAD)
+
+[Full Changelog](https://github.com/gjtorikian/html-pipeline/compare/v3.0.0.pre3...HEAD)
+
+**Closed issues:**
+
+- v3: Question regarding requiring a ConvertFilter if there are NodeFilters [\#374](https://github.com/gjtorikian/html-pipeline/issues/374)
+
+## [v3.0.0.pre3](https://github.com/gjtorikian/html-pipeline/tree/v3.0.0.pre3) (2023-02-15)
+
+[Full Changelog](https://github.com/gjtorikian/html-pipeline/compare/v3.0.0.pre2...v3.0.0.pre3)
+
+**Merged pull requests:**
+
+- req convert\_filter if `text/node`filter present [\#375](https://github.com/gjtorikian/html-pipeline/pull/375) ([gjtorikian](https://github.com/gjtorikian))
+
+## [v3.0.0.pre2](https://github.com/gjtorikian/html-pipeline/tree/v3.0.0.pre2) (2023-01-26)
+
+[Full Changelog](https://github.com/gjtorikian/html-pipeline/compare/v3.0.0.pre1...v3.0.0.pre2)
+
+**Closed issues:**
+
+- Indicate a version for activesupport that has support/receives security patches \(\>= 6?\) [\#367](https://github.com/gjtorikian/html-pipeline/issues/367)
+- Add MathML elements to whitelist [\#336](https://github.com/gjtorikian/html-pipeline/issues/336)
+- Feature request: add safe semantic HTML tags to default whitelist [\#312](https://github.com/gjtorikian/html-pipeline/issues/312)
+- Allow float: left|right and clear: left|right|both in sanitation [\#302](https://github.com/gjtorikian/html-pipeline/issues/302)
+- Open link in new tab option [\#266](https://github.com/gjtorikian/html-pipeline/issues/266)
+- Consider allowing LINK and META elements in HTML [\#261](https://github.com/gjtorikian/html-pipeline/issues/261)
+- Allow SVG elements in whitelist [\#251](https://github.com/gjtorikian/html-pipeline/issues/251)
+- Allow RDFa 1.1 \(Lite\) attributes [\#249](https://github.com/gjtorikian/html-pipeline/issues/249)
+- What’s the point of allowing the accept-charset attribute in the sanitization filter ? [\#218](https://github.com/gjtorikian/html-pipeline/issues/218)
+- Use gemojione instead of gemoji [\#200](https://github.com/gjtorikian/html-pipeline/issues/200)
+- Link schema not used with \<a\> markup : print them directly [\#194](https://github.com/gjtorikian/html-pipeline/issues/194)
+
+**Merged pull requests:**
+
+- Use emoji from commonmarker [\#373](https://github.com/gjtorikian/html-pipeline/pull/373) ([gjtorikian](https://github.com/gjtorikian))
+
 ## [v3.0.0.pre1](https://github.com/gjtorikian/html-pipeline/tree/v3.0.0.pre1) (2022-12-30)
 
 [Full Changelog](https://github.com/gjtorikian/html-pipeline/compare/v2.14.3...v3.0.0.pre1)

data/Gemfile

@@ -27,9 +27,10 @@ group :development do
 end
 
 group :test do
-  gem "commonmarker", "~> 1.0.0.pre4", require: false
+  gem "commonmarker", "~> 1.0.0.pre7", require: false
   gem "gemoji", "~> 3.0", require: false
   gem "gemojione", "~> 4.3", require: false
+
   gem "minitest"
 
   gem "minitest-bisect", "~> 1.6"
@@ -37,4 +38,5 @@ group :test do
   gem "nokogiri", "~> 1.13"
 
   gem "minitest-focus", "~> 1.1"
+  gem "rouge", "~> 3.1", require: false
 end

data/README.md

@@ -230,19 +230,28 @@ end
 
 For more information on how to write effective `NodeFilter`s, refer to the provided filters, and see the underlying lib, [Selma](https://www.github.com/gjtorikian/selma) for more information.
 
-- `AbsoluteSourceFilter` - replace relative image urls with fully qualified versions
-- `EmojiFilter` - converts `:<emoji>:` to [emoji](http://www.emoji-cheat-sheet.com/)!
-- `HttpsFilter` - Replacing http urls with https versions
-- `ImageMaxWidthFilter` - link to full size image for large images
-- `MentionFilter` - replace `@user` mentions with links
-- `SanitizationFilter` - allow sanitize user markup
-- `TableOfContentsFilter` - anchor headings with name attributes and generate Table of Contents html unordered list linking headings
-- `TeamMentionFilter` - replace `@org/team` mentions with links
+- `AbsoluteSourceFilter`: replace relative image urls with fully qualified versions
+- `EmojiFilter`: converts `:<emoji>:` to [emoji](http://www.emoji-cheat-sheet.com/)
+  - (Note: the included `MarkdownFilter` will already convert emoji)
+- `HttpsFilter`: Replacing http urls with https versions
+- `ImageMaxWidthFilter`: link to full size image for large images
+- `MentionFilter`: replace `@user` mentions with links
+- `SanitizationFilter`: allow sanitize user markup
+- `SyntaxHighlightFilter`: applies syntax highlighting to `pre` blocks
+  - (Note: the included `MarkdownFilter` will already apply highlighting)
+- `TableOfContentsFilter`: anchor headings with name attributes and generate Table of Contents html unordered list linking headings
+- `TeamMentionFilter`: replace `@org/team` mentions with links
 
 ## Dependencies
 
-Since filters can be customized to your heart's content, gem dependencies are _not_ bundled; this project doesn't know which of the default filters you might use, and as such, you must bundle each filter's gem
-dependencies yourself.
+Since filters can be customized to your heart's content, gem dependencies are _not_ bundled; this project doesn't know which of the default filters you might use, and as such, you must bundle each filter's gem dependencies yourself.
+
+For example, `SyntaxHighlightFilter` uses [rouge](https://github.com/jneen/rouge)
+to detect and highlight languages; to use the `SyntaxHighlightFilter`, you must add the following to your Gemfile:
+
+```ruby
+gem "rouge"
+```
 
 > **Note**
 > See the [Gemfile](/Gemfile) `:test` group for any version requirements.

data/UPGRADING.md

@@ -13,7 +13,6 @@ This project is now under a module called `HTMLPipeline`, not `HTML::Pipeline`.
 The following filters were removed:
 
 - `AutolinkFilter`: this is handled by [Commonmarker](https://www.github.com/gjtorikian/commonmarker) and can be disabled/enabled through the `MarkdownFilter`'s `context` hash
-- `SyntaxHighlightFilter`: this is handled by [Commonmarker](https://www.github.com/gjtorikian/commonmarker) and can be disabled/enabled through the `MarkdownFilter`'s `context` hash
 - `SanitizationFilter`: this is handled by [Selma](https://www.github.com/gjtorikian/selma); configuration can be done through the `sanitization_config` hash
 
 - `EmailReplyFilter`

data/lib/html_pipeline/filter.rb

@@ -80,7 +80,7 @@ class HTMLPipeline
     def needs(*keys)
       missing = keys.reject { |key| context.include?(key) }
 
-      return unless missing.any?
+      return if missing.none?
 
       raise ArgumentError,
         "Missing context keys for #{self.class.name}: #{missing.map(&:inspect).join(", ")}"

data/lib/html_pipeline/node_filter/syntax_highlight_filter.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+HTMLPipeline.require_dependency("rouge", "SyntaxHighlightFilter")
+
+class HTMLPipeline
+  class NodeFilter
+    # HTML Filter that syntax highlights text inside code blocks.
+    #
+    # Context options:
+    #
+    #   :highlight => String represents the language to pick lexer. Defaults to empty string.
+    #   :scope => String represents the class attribute adds to pre element after.
+    #             Defaults to "highlight highlight-css" if highlights a css code block.
+    #
+    # This filter does not write any additional information to the context hash.
+    class SyntaxHighlightFilter < NodeFilter
+      def initialize(context: {}, result: {})
+        super(context: context, result: result)
+        # TODO: test the optionality of this
+        @formatter = context[:formatter] || Rouge::Formatters::HTML.new
+      end
+
+      SELECTOR = Selma::Selector.new(match_element: "pre", match_text_within: "pre")
+
+      def selector
+        SELECTOR
+      end
+
+      def handle_element(element)
+        default = context[:highlight]&.to_s
+        @lang = element["lang"] || default
+
+        scope = context.fetch(:scope, "highlight")
+
+        element["class"] = "#{scope} #{scope}-#{@lang}" if include_lang?
+      end
+
+      def handle_text_chunk(text)
+        return if @lang.nil?
+        return if (lexer = lexer_for(@lang)).nil?
+
+        content = text.to_s
+
+        text.replace(highlight_with_timeout_handling(content, lexer), as: :html)
+      end
+
+      def highlight_with_timeout_handling(text, lexer)
+        Rouge.highlight(text, lexer, @formatter)
+      rescue Timeout::Error => _e
+        text
+      end
+
+      def lexer_for(lang)
+        Rouge::Lexer.find(lang)
+      end
+
+      def include_lang?
+        !@lang.nil? && !@lang.empty?
+      end
+    end
+  end
+end

data/lib/html_pipeline/node_filter/table_of_contents_filter.rb

@@ -24,8 +24,10 @@ class HTMLPipeline
     #  result[:output].to_s
     #  # => "<h1>\n<a id=\"ice-cube\" class=\"anchor\" href=\"#ice-cube\">..."
     class TableOfContentsFilter < NodeFilter
-      SELECTOR = Selma::Selector.new(match_element: "h1 a[href], h2 a[href], h3 a[href], h4 a[href], h5 a[href], h6 a[href]",
-        match_text_within: "h1, h2, h3, h4, h5, h6")
+      SELECTOR = Selma::Selector.new(
+        match_element: "h1 a[href], h2 a[href], h3 a[href], h4 a[href], h5 a[href], h6 a[href]",
+        match_text_within: "h1, h2, h3, h4, h5, h6",
+      )
 
       def selector
         SELECTOR

data/lib/html_pipeline/sanitization_filter.rb

@@ -16,11 +16,70 @@ class HTMLPipeline
     # The main sanitization allowlist. Only these elements and attributes are
     # allowed through by default.
     DEFAULT_CONFIG = Selma::Sanitizer::Config.freeze_config({
-      elements: ["h1", "h2", "h3", "h4", "h5", "h6", "br", "b", "i", "strong", "em", "a", "pre", "code",
-                 "img", "tt", "div", "ins", "del", "sup", "sub", "p", "picture", "ol", "ul", "table", "thead", "tbody", "tfoot",
-                 "blockquote", "dl", "dt", "dd", "kbd", "q", "samp", "var", "hr", "ruby", "rt", "rp", "li", "tr", "td", "th",
-                 "s", "strike", "summary", "details", "caption", "figure", "figcaption", "abbr", "bdo", "cite",
-                 "dfn", "mark", "small", "source", "span", "time", "wbr",],
+      elements: [
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        "br",
+        "b",
+        "i",
+        "strong",
+        "em",
+        "a",
+        "pre",
+        "code",
+        "img",
+        "tt",
+        "div",
+        "ins",
+        "del",
+        "sup",
+        "sub",
+        "p",
+        "picture",
+        "ol",
+        "ul",
+        "table",
+        "thead",
+        "tbody",
+        "tfoot",
+        "blockquote",
+        "dl",
+        "dt",
+        "dd",
+        "kbd",
+        "q",
+        "samp",
+        "var",
+        "hr",
+        "ruby",
+        "rt",
+        "rp",
+        "li",
+        "tr",
+        "td",
+        "th",
+        "s",
+        "strike",
+        "summary",
+        "details",
+        "caption",
+        "figure",
+        "figcaption",
+        "abbr",
+        "bdo",
+        "cite",
+        "dfn",
+        "mark",
+        "small",
+        "source",
+        "span",
+        "time",
+        "wbr",
+      ],
 
       attributes: {
         "a" => ["href"],
@@ -31,13 +90,77 @@ class HTMLPipeline
         "ins" => ["cite"],
         "q" => ["cite"],
         "source" => ["srcset"],
-        all: ["abbr", "accept", "accept-charset", "accesskey", "action", "align", "alt", "aria-describedby",
-              "aria-hidden", "aria-label", "aria-labelledby", "axis", "border", "char",
-              "charoff", "charset", "checked", "clear", "cols", "colspan", "compact", "coords", "datetime", "dir",
-              "disabled", "enctype", "for", "frame", "headers", "height", "hreflang", "hspace", "id", "ismap", "label", "lang",
-              "maxlength", "media", "method", "multiple", "name", "nohref", "noshade", "nowrap", "open", "progress",
-              "prompt", "readonly", "rel", "rev", "role", "rows", "rowspan", "rules", "scope", "selected", "shape",
-              "size", "span", "start", "summary", "tabindex",  "title", "type", "usemap", "valign", "value", "width", "itemprop",],
+        all: [
+          "abbr",
+          "accept",
+          "accept-charset",
+          "accesskey",
+          "action",
+          "align",
+          "alt",
+          "aria-describedby",
+          "aria-hidden",
+          "aria-label",
+          "aria-labelledby",
+          "axis",
+          "border",
+          "char",
+          "charoff",
+          "charset",
+          "checked",
+          "clear",
+          "cols",
+          "colspan",
+          "compact",
+          "coords",
+          "datetime",
+          "dir",
+          "disabled",
+          "enctype",
+          "for",
+          "frame",
+          "headers",
+          "height",
+          "hreflang",
+          "hspace",
+          "id",
+          "ismap",
+          "label",
+          "lang",
+          "maxlength",
+          "media",
+          "method",
+          "multiple",
+          "name",
+          "nohref",
+          "noshade",
+          "nowrap",
+          "open",
+          "progress",
+          "prompt",
+          "readonly",
+          "rel",
+          "rev",
+          "role",
+          "rows",
+          "rowspan",
+          "rules",
+          "scope",
+          "selected",
+          "shape",
+          "size",
+          "span",
+          "start",
+          "summary",
+          "tabindex",
+          "title",
+          "type",
+          "usemap",
+          "valign",
+          "value",
+          "width",
+          "itemprop",
+        ],
       },
       protocols: {
         "a" => { "href" => Selma::Sanitizer::Config::VALID_PROTOCOLS }.freeze,

data/lib/html_pipeline/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class HTMLPipeline
-  VERSION = "3.0.0.pre1"
+  VERSION = "3.0.0.pre3"
 end

data/lib/html_pipeline.rb

@@ -116,8 +116,9 @@ end
     validate_filters(@node_filters, HTMLPipeline::NodeFilter)
 
     @convert_filter = convert_filter
-    if @convert_filter.nil? && !@node_filters.empty?
-      raise InvalidFilterError, "Must provide `convert_filter` if `node_filter`s is also provided"
+
+    if @convert_filter.nil? && (!@text_filters.empty? && !@node_filters.empty?)
+      raise InvalidFilterError, "Must provide `convert_filter` if `text_filters` and `node_filters` are also provided"
     elsif !@convert_filter.nil?
       validate_filter(@convert_filter, HTMLPipeline::ConvertFilter)
     end
@@ -145,8 +146,11 @@ end
     context = context.freeze
     result ||= {}
 
-    payload = default_payload({ text_filters: @text_filters.map(&:name),
-                                context: context, result: result, })
+    payload = default_payload({
+      text_filters: @text_filters.map(&:name),
+      context: context,
+      result: result,
+    })
     instrument("call_text_filters.html_pipeline", payload) do
       result[:output] =
         @text_filters.inject(text) do |doc, filter|
@@ -159,8 +163,11 @@ end
     html = @convert_filter.call(text) unless @convert_filter.nil?
 
     unless @node_filters.empty?
-      payload = default_payload({ node_filters: @node_filters.map { |f| f.class.name },
-        context: context, result: result, })
+      payload = default_payload({
+        node_filters: @node_filters.map { |f| f.class.name },
+        context: context,
+        result: result,
+      })
       instrument("call_node_filters.html_pipeline", payload) do
         result[:output] = Selma::Rewriter.new(sanitizer: @sanitization_config, handlers: @node_filters).rewrite(html)
       end
@@ -178,8 +185,11 @@ end
   #
   # Returns the result of the filter.
   def perform_filter(filter, doc, context: {}, result: {})
-    payload = default_payload({ filter: filter.name,
-                                context: context, result: result, })
+    payload = default_payload({
+      filter: filter.name,
+      context: context,
+      result: result,
+    })
     instrument("call_filter.html_pipeline", payload) do
       filter.call(doc, context: context, result: result)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: html-pipeline
 version: !ruby/object:Gem::Version
-  version: 3.0.0.pre1
+  version: 3.0.0.pre3
 platform: ruby
 authors:
 - Garen J. Torikian
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-12-30 00:00:00.000000000 Z
+date: 2023-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selma
@@ -71,6 +71,7 @@ files:
 - lib/html_pipeline/node_filter/https_filter.rb
 - lib/html_pipeline/node_filter/image_max_width_filter.rb
 - lib/html_pipeline/node_filter/mention_filter.rb
+- lib/html_pipeline/node_filter/syntax_highlight_filter.rb
 - lib/html_pipeline/node_filter/table_of_contents_filter.rb
 - lib/html_pipeline/node_filter/team_mention_filter.rb
 - lib/html_pipeline/sanitization_filter.rb