mato 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc964ed48f7d5e7d1d069e32b2ea3701118a6765
+  data.tar.gz: 310ca6e0226f7ebaf1da5653d17a926580c8c476
+SHA512:
+  metadata.gz: b19a063ae37c6885ae08a1e1d961a3519f36dc19fd42370a09f95a143f5a1b3f84d5d17adb17f48e8247db268d8750f0c1759a54193e6f7c93e73696a08b45ba
+  data.tar.gz: 1b6d1995d00dba709bd52d8138530ce8fe66699b5006060dc10277333569a23582fec03ab0f86a47380e3eee5328fbc6739b841152d5a5a27219d3f842ffca80

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea/

data/.rubocop.yml

@@ -0,0 +1,139 @@
+# See also https://github.com/onk/onkcop/blob/master/config/rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 2.3
+  DisplayCopNames: true
+  Exclude:
+    - bin/**/*
+
+Lint/UnusedBlockArgument:
+  Enabled: false
+
+Lint/AmbiguousBlockAssociation:
+  Enabled: false
+
+Lint/ScriptPermission:
+  Enabled: false
+
+Layout/MultilineMethodCallIndentation:
+  Enabled: false
+
+Layout/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Metrics/AbcSize:
+  Max: 32
+
+Metrics/PerceivedComplexity:
+  Max: 10
+
+Metrics/CyclomaticComplexity:
+  Max: 10
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Max: 45
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/ParameterLists:
+  Enabled: false
+
+Style/BlockDelimiters:
+  Enabled: false
+
+Style/CommentAnnotation:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/FileName:
+  Enabled: false
+
+Style/GuardClause:
+  Enabled: false
+
+Style/IfInsideElse:
+  Enabled: false
+
+Style/IfUnlessModifier:
+  Enabled: false
+
+Style/Lambda:
+  Enabled: false
+
+Style/MutableConstant:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Style/PercentLiteralDelimiters:
+  Enabled: false
+
+Style/SignalException:
+  Enabled: false
+
+Style/StringLiterals:
+  Enabled: false
+
+Style/TrailingCommaInLiteral:
+  EnforcedStyleForMultiline: comma
+
+Style/TrailingCommaInArguments:
+  EnforcedStyleForMultiline: comma
+
+Style/WordArray:
+  Enabled: false
+
+Style/NumericLiterals:
+  Enabled: false
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Style/ConditionalAssignment:
+  Enabled: false
+
+Style/RescueModifier:
+  Enabled: false
+
+Style/AsciiComments:
+  Enabled: false
+
+Style/EmptyMethod:
+  EnforcedStyle: expanded
+
+Style/VariableNumber:
+  Enabled: false
+
+Style/PredicateName:
+  Enabled: false
+
+Style/AccessorMethodName:
+  Enabled: false
+
+Style/YodaCondition:
+  Enabled: false
+
+Style/FormatStringToken:
+  Enabled: false
+
+Style/MultipleComparison:
+  Enabled: false
+
+Style/StructInheritance:
+  Enabled: false
+
+Performance/RedundantBlockCall:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,9 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3
+  - 2.4
+before_install: gem install bundler
+notifications:
+  slack:
+    secure: W/CuFdMffK19NqZ7hX5Cx1ekLwY5WYtyi91CgYJ5jUYZq6hY7sZAHItfvbB7kwVc5vGzDVzBp1L4+iR44OX1+3sbJQ8OHUfWNksRME5NAHeHKUnkeoMAToTy6gv8GTSDOkBoMxgtZQQwaFFxOVQF4UknLCKvwz1JRBf3zjEG7zyKlBQuYSkMQQm4xA/qmgb6nCAqSzMJxN4Gl57LKphCScKO5ZK6hUuaJv4H6y3NfDbKAEwidEUaMqnU1TtLTkqhSqVve/JMlBoynYFIezBu7R35cyHFdE8l1OaYFvIJo6nsyRZr9RVhp20wlLyL2dsM4IUBTfLf1dCw94U8VE3cCkYEh7MxQv/NAkDpkBsI9E4GGOfmuXRxcoQGoKRI9GHleg8hm/E39134JhVKNL2L5RGg/8ULNPPrQjMf0GWSxhtqBXN7vzuP4ZfAMzEDDGFehCmmzrE5qIdcRwXCI3+QRX7XAWL+Ql+xljoxZjl2f7wHSx3COChkol0V8CEGAyOTJc3qnci5jPs5mSByqv3RyO0meYiJCJ1Ce2FTjCOHs1G9D58xfra6AVA8FmOk43+iN8QZyUIqd+MsenL2k89LqMijoC208xI6WI88C9tMHDA8tmi8yJt6PA00emC+Gzy9xPCYsZ+dZhymOPDcAbBlez+j3weEEDROF0Jlewt4X8Q=

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mato.gemspec
+gemspec
+
+gem "sanitize"
+gem "rouge"
+
+gem "m"
+gem "minitest"
+gem "minitest-power_assert"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 FUJI Goro (gfx)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,104 @@
+# Mato - Markdown Toolkit based on CommonMark [![Build Status](https://travis-ci.org/bitjourney/mato.svg?branch=master)](https://travis-ci.org/bitjourney/mato)
+
+**Mato**, standing for **Ma**rkdown **To**oolkit,  is an extensible markdown-based content processing toolkit, inspired by [HTML::Pipeline](https://github.com/jch/html-pipeline).
+
+
+This gem is built on [commonmarker](https://github.com/gjtorikian/commonmarker), a [CommonMark](https://github.com/jgm/CommonMark) implementation,
+which can parse markdown documents into AST.
+
+## Synopsis
+
+```ruby
+require 'mato'
+
+# define a markdown-to-html processor:
+mato = Mato.define do |config|
+  # append pre-defined HTML filters:
+  config.append_html_filter(Mato::HtmlFilters::SyntaxHighlight.new)
+  config.append_html_filter(Mato::HtmlFilters::TaskList.new)
+  config.append_html_filter(Mato::HtmlFilters::SectionAnchor.new)
+
+  # append MentionLink, a customizable HTML filter:
+  config.append_html_filter(Mato::HtmlFilters::MentionLink.new do |mention_candidate_map|
+    candidate_accounts = mention_candidate_map.keys.map { |name| name.gsub(/^\@/, '') }
+    User.where(account: candidate_accounts).each do |user|
+      mention = "@#{user.account}"
+      mention_candidate_map[mention].each do |node|
+        node.replace("<a href='https://twitter.com/#{mention}' class='mention'>#{mention}</a>")
+      end
+    end
+  end)
+end
+
+# Prosesses markdown into Mato::Document:
+doc = mato.process(markdown_content)
+
+# Renders doc as HTML:
+html = doc.render_html
+
+# Renders doc as HTML Table of Contents:
+html_toc = doc.render_html_toc
+
+# Extracts elements (e.g. mentions) with CSS selector:
+doc.css('a').each do |element|
+  # do something with element: Nokogiri::XML::Element
+end
+
+# Extracts nodes (e.g. mentions) with XPath selector:
+doc.xpath('./text()').each do |node|
+  # do something with node: Nokogiri::XML::Text
+end
+
+# Mato::Document can be cached with Rails.cache (i.e. Marshal.dump ready)
+Rails.fetch(digest(markdown_content)) do
+  mato.process(markdown_content)
+end
+
+# Applies extra filters and returns a new Mato::Document.
+# Because Mato::Document is serializable, you can cache the base doc and then apply extra filters on demaond.
+new_doc = doc.apply_html_filters(
+  -> (fragment) { modify_fragment!(fragment) },
+  SomeHtmlFilter.new, # anything that has #call(node) method
+)
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby:Gemfile
+gem 'mato'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mato
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Optional Dependencies
+
+* [rouge](https://github.com/jneen/rouge) (>= 2.0) to use `SyntaxHighlight`
+* [sanitize](https://github.com/rgrove/sanitize) (>= 4.0) to use `Sanitization`
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/bitjourney/mato.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.warning = false
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task default: :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mato"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example/hello.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'mato'
+
+# User class that mocks ActiveRecord's
+User = Struct.new(:account)
+class User
+  VALID_ACCOUNTS = %w(mato)
+
+  # @return [Enumerable<User>]
+  def self.where(account:)
+    (account & VALID_ACCOUNTS).map do |name|
+      User.new(name)
+    end
+  end
+end
+
+mato = Mato.define do |config|
+  config.append_text_filter ->(text, _context) {
+    # weave text
+  }
+
+  config.append_markdown_filter ->(doc, _context) {
+    # weave doc
+  }
+
+  config.append_html_filter ->(doc, _context) {
+    # weave doc
+  }
+
+  config.append_html_filter(Mato::HtmlFilters::TaskList.new)
+  config.append_html_filter(Mato::HtmlFilters::MentionLink.new do |mention_candidate_map|
+    candidate_accounts = mention_candidate_map.keys.map { |name| name.gsub(/^\@/, '') }
+    User.where(account: candidate_accounts).each do |user|
+      mention = "@#{user.account}"
+      mention_candidate_map[mention].each do |node|
+        node.replace("<a href='https://twitter.com/#{mention}' class='mention'>#{mention}</a>")
+      end
+    end
+  end)
+end
+
+puts mato.process(<<~'MARKDOWN').render_html
+  # Hello, @mato!
+
+  @this_is_invalid_mention
+
+  https://twitter.com/@kibe_la
+
+  * [ ] a
+  * [x] b
+MARKDOWN

data/example/toc.rb

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'mato'
+
+mato = Mato.define do |config|
+  config.append_html_filter(Mato::HtmlFilters::SectionAnchor.new)
+end
+
+puts mato.process(<<~'MARKDOWN').render_html_toc
+  # **First** Level Title
+
+  ## **Second** Level Title
+
+  ### **Third** Level Title
+
+  ##### **Fifth** Level Title
+
+  # **First** Level Title Again
+MARKDOWN

data/lib/mato.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# core classes
+require_relative "./mato/version"
+require_relative "./mato/config"
+require_relative "./mato/processor"
+
+# filter classes
+require_relative "./mato/html_filters/token_link"
+require_relative "./mato/html_filters/mention_link"
+require_relative "./mato/html_filters/syntax_highlight"
+require_relative "./mato/html_filters/task_list"
+require_relative "./mato/html_filters/section_anchor"
+require_relative "./mato/html_filters/sanitization"
+
+module Mato
+  # @param [Proc] block
+  # @yieldparam [Mato::Config] config
+  # @return [Mato::Processor]
+  def self.define(&block)
+    config = Mato::Config.new
+    config.configure(&block)
+    Mato::Processor.new(config)
+  end
+end

data/lib/mato/anchor_builder.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'erb'
+
+module Mato
+  class AnchorBuilder
+
+    # assumes use of font-awesome
+    # specify it as "<span aria-hidden=\"true\" class=\"octicon octicon-link\"></span>" if you use octicon
+    DEFAULT_ANCHOR_ICON_ELEMENT = '<i class="fa fa-link"></i>'
+
+    CSS_CLASS_NAME = "anchor"
+
+    attr_reader :anchor_icon_element
+    attr_reader :context
+
+    def initialize(anchor_icon_element = DEFAULT_ANCHOR_ICON_ELEMENT)
+      @anchor_icon_element = anchor_icon_element
+      @id_map = {}
+    end
+
+    # @param [Nokogiri::XML::Node] hx
+    def make_anchor_element(hx)
+      id = make_anchor_id(hx)
+      %{<a id="#{id}" href="##{id}" aria-hidden="true" class="#{CSS_CLASS_NAME}">#{anchor_icon_element}</a>}
+    end
+
+    def make_anchor_id(hx)
+      prefix = make_anchor_id_prefix(hx.inner_text)
+      "#{prefix}#{make_anchor_id_suffix(prefix)}"
+    end
+
+    private
+
+    def make_anchor_id_suffix(text)
+      @id_map[text] ||= -1
+      unique_id = @id_map[text] += 1
+
+      if unique_id > 0
+        "-#{unique_id}"
+      else
+        ""
+      end
+    end
+
+    def make_anchor_id_prefix(text)
+      ERB::Util.url_encode(text.downcase.gsub(/[^\p{Word}\- ]/u, "").tr(" ", "-"))
+    end
+  end
+end

data/lib/mato/concerns/html_node_checkable.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Mato
+  module Concerns
+    module HtmlNodeCheckable
+      module_function
+
+      # @param [Nokogiri::XML::Node] node
+      # @param [Array<String>] tags - set of tags
+      # @return [Boolean] true if the node has the specified tags as a parent
+      def has_ancestor?(node, *tags)
+        current = node
+        while (current = current.parent)
+          if tags.include?(current.name)
+            return true
+          end
+        end
+        false
+      end
+    end
+  end
+end

data/lib/mato/config.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative('./document')
+
+require 'nokogiri'
+
+module Mato
+  class Config
+    # https://github.com/gjtorikian/commonmarker#parse-options
+    DEFAULT_MARKDOWN_PARSE_OPTIONS = %i[
+      DEFAULT
+      VALIDATE_UTF8
+    ]
+
+    # https://github.com/gjtorikian/commonmarker#render-options
+    DEFAULT_MARKDOWN_RENDER_OPTIONS = [
+      :DEFAULT,
+      :HARDBREAKS, # convert "\n" as <br/>
+      # :SOURCEPOS, // TODO: enable it after assertions are supported
+    ]
+
+    # https://github.com/github/cmark/tree/master/extensions
+    DEFAULT_MARKDOWN_EXTENSIONS = %i[
+      table
+      strikethrough
+      autolink
+      tagfilter
+    ]
+
+    # @return [Array<Proc>]
+    attr_accessor :text_filters
+
+    # @return [Array<Proc>]
+    attr_accessor :markdown_filters
+
+    # @return [Array<Proc>]
+    attr_accessor :html_filters
+
+    # @return [Class<CommonMarker]
+    attr_accessor :markdown_parser
+
+    # @return [Cass<Nokogiri::HTML::DocumentFragment>]
+    attr_accessor :html_parser
+
+    # @return [Class<Mato::Document>]
+    attr_accessor :document_factory
+
+    # @return [Array<Symbol>] CommonMarker's parse extensions
+    attr_accessor :markdown_extensions
+
+    # @return [Array<Symbol>] CommonMarker's pars options
+    attr_accessor :markdown_parse_options
+
+    # @return [Array<Symbol>] CommonMarker's HTML rendering options
+    attr_accessor :markdown_render_options
+
+    def initialize
+      @text_filters = []
+      @markdown_filters = []
+      @html_filters = []
+
+      @markdown_parser = CommonMarker
+      @html_parser = Nokogiri::HTML::DocumentFragment
+
+      @document_factory = Document
+
+      @markdown_extensions = DEFAULT_MARKDOWN_EXTENSIONS
+      @markdown_parse_options = DEFAULT_MARKDOWN_PARSE_OPTIONS
+      @markdown_render_options = DEFAULT_MARKDOWN_RENDER_OPTIONS
+    end
+
+    # @param [Proc] block
+    # @yieldparam [Mato::Config] config
+    def configure(&block)
+      block.call(self)
+    end
+
+    def append_text_filter(text_filter)
+      raise "text_filter must respond to call()" unless text_filter.respond_to?(:call)
+      text_filters.push(text_filter)
+    end
+
+    def prepend_text_filter(text_filter)
+      raise "text_filter must respond to call()" unless text_filter.respond_to?(:call)
+      text_filters.unshift(text_filter)
+    end
+
+    def append_markdown_filter(markdown_filter)
+      raise "markdown_filter must respond to call()" unless markdown_filter.respond_to?(:call)
+      markdown_filters.push(markdown_filter)
+    end
+
+    def prepend_markdown_filter(markdown_filter)
+      raise "markdown_filter must respond to call()" unless markdown_filter.respond_to?(:call)
+      markdown_filters.unshift(markdown_filter)
+    end
+
+    def append_html_filter(html_filter)
+      raise "html_filter must respond to call()" unless html_filter.respond_to?(:call)
+      html_filters.push(html_filter)
+    end
+
+    def prepend_html_filter(html_filter)
+      raise "html_filter must respond to call()" unless html_filter.respond_to?(:call)
+      html_filters.unshift(html_filter)
+    end
+  end
+end

data/lib/mato/document.rb

@@ -0,0 +1,65 @@
+
+# frozen_string_literal: true
+
+require_relative './renderers/html_renderer'
+require_relative './renderers/html_toc_renderer'
+
+# Intermediate document class, which instance is *serializable*.
+module Mato
+  class Document
+    # @return [Nokogiri::HTML::DocumentFragment]
+    attr_reader :fragment
+
+    def self.empty
+      new(Nokogiri::HTML.fragment(''))
+    end
+
+    # @param [Nokogiri::HTML::DocumentFragment] fragment
+    def initialize(fragment)
+      @fragment = fragment
+    end
+
+    # @return [Nokogiri::HTML::DocumentFragment] A copy of fragment that are modified by html_filters
+    def apply_html_filters(*html_filters)
+      new_fragment = fragment.dup
+      html_filters.each do |html_filter|
+        html_filter.call(new_fragment)
+      end
+      self.class.new(new_fragment.freeze)
+    end
+
+    # @param [String] selector
+    # @return [Nokogiri::XML::NodeSet]
+    def css(selector)
+      fragment.css(selector)
+    end
+
+    # @param [String] query
+    # @return [Nokogiri::XML::NodeSet]
+    def xpath(query)
+      fragment.xpath(query)
+    end
+
+    def render(renderer)
+      renderer.call(fragment)
+    end
+
+    def render_html
+      render(Mato::Renderers::HtmlRenderer.new)
+    end
+
+    def render_html_toc
+      render(Mato::Renderers::HtmlTocRenderer.new)
+    end
+
+    def marshal_dump
+      {
+        fragment: fragment.to_s,
+      }
+    end
+
+    def marshal_load(data)
+      initialize(Nokogiri::HTML.fragment(data[:fragment]).freeze)
+    end
+  end
+end

data/lib/mato/html_filters/mention_link.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/html_node_checkable'
+
+module Mato
+  module HtmlFilters
+    class MentionLink
+      include Concerns::HtmlNodeCheckable
+
+      # @return [Regexp]
+      attr_reader :pattern
+
+      # @return [Proc]
+      attr_reader :link_builder
+
+      MENTION_PATTERN = /\@[a-zA-Z0-9_]+\b/ # e.g. @foo
+
+      # @param [Regexp] pattern
+      # @param [Proc] link_builder A block that takes Hash<String, Array<Nokogiri::XML::Node>>
+      def initialize(pattern = MENTION_PATTERN, &link_builder)
+        @pattern = pattern
+        @link_builder = link_builder
+      end
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        candidate_map = {}
+        candidates = []
+
+        doc.xpath('.//text()').each do |text_node|
+          next if has_ancestor?(text_node, 'a', 'code', 'pre')
+
+          candidate_html = text_node.content.gsub(pattern) do |mention|
+            "<span class='mention-candidate'>#{mention}</span>"
+          end
+
+          next if text_node.content == candidate_html
+
+          candidate_fragment = text_node.replace(candidate_html)
+          candidate_fragment.css('span').each do |mention_element|
+            (candidate_map[mention_element.child.content] ||= []) << mention_element
+          end
+
+          candidates << candidate_fragment
+        end
+
+        unless candidate_map.empty?
+          link_builder.call(candidate_map)
+
+          # cleanup
+          candidates.each do |candidate_fragment|
+            candidate_fragment.css('span.mention-candidate').each do |node|
+              node.replace(node.child.content)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mato/html_filters/sanitization.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# note: do require 'sanitize' by yourself
+
+# https://github.com/rgrove/sanitize
+module Mato
+  module HtmlFilters
+    class Sanitization
+      attr_reader :sanitize
+
+      # @param [Sanitize] sanitize
+      def initialize(sanitize:)
+        @sanitize = sanitize
+      end
+
+      def call(doc)
+        sanitize.node!(doc)
+      end
+    end
+  end
+end

data/lib/mato/html_filters/section_anchor.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative '../anchor_builder'
+
+module Mato
+  module HtmlFilters
+    class SectionAnchor
+
+      HX_PATTERN = 'h1,h2,h3,h4,h5,h6'
+
+      def initialize(anchor_icon_element = AnchorBuilder::DEFAULT_ANCHOR_ICON_ELEMENT)
+        @anchor_icon_element = anchor_icon_element
+      end
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        anchor_builder = AnchorBuilder.new(@anchor_icon_element)
+
+        doc.css(HX_PATTERN).each do |hx|
+          hx.children = anchor_builder.make_anchor_element(hx) + hx.children.to_html
+        end
+      end
+    end
+  end
+end

data/lib/mato/html_filters/syntax_highlight.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+# do require 'rouge' by yourself
+
+module Mato
+  module HtmlFilters
+    class SyntaxHighlight
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        doc.search("pre").each do |pre|
+          if pre.at('code')
+            pre.replace(highlight(pre))
+          end
+        end
+      end
+
+      # @param [String] language
+      # @param [String] filename
+      # @param [String] source
+      # @return [Rouge::Lexer]
+      def guess_lexer(language, filename, source)
+        Rouge::Lexer.find(language)&.tap do |lexer|
+          return lexer.new
+        end
+
+        lexers = Rouge::Lexer.guesses(filename: filename, source: source)
+
+        if lexers.empty?
+          Rouge::Lexers::PlainText.new
+        else
+          lexers.first.new
+        end
+      end
+
+      # @param [String,nil] CSS class names, e.g. "foo.js" "ruby:foo.rb"
+      def parse_label(class_name)
+        a = class_name&.split(/:/) || []
+        if a.empty?
+          {}
+        elsif a.size == 1
+          token = a[0].sub(/^language-/, '')
+          if Rouge::Lexer.find(token)
+            { language: token }
+          else
+            { filename: token }
+          end
+        else
+          { language: a[0], filename: a[1] }
+        end
+      end
+
+      private
+
+      # @param [Nokogiri::XML::Element] pre a <pre/> element
+      # @return [Nokogiri::XML::Element] a new <div/> wrapping the given code block
+      def highlight(pre)
+        code = pre.at('code')
+        metadata = parse_label(code['class'])
+        language = metadata[:language]
+        filename = metadata[:filename]
+        source = code.inner_text
+
+        lexer = guess_lexer(language, filename, source)
+
+        document = Nokogiri::HTML.fragment(%{<div class="code-frame"/>})
+
+        div = document.at('div')
+        div.add_child(label_fragment(filename || language || lexer.tag)) if filename || !lexer.is_a?(Rouge::Lexers::PlainText)
+        div.add_child(%{<pre class="highlight"><code data-lang="#{lexer.tag}">#{format(lexer, source)}</code></pre>})
+
+        document
+      end
+
+      def label_fragment(label)
+        Nokogiri::HTML.fragment(%{<div class="code-label"/>}).tap do |fragment|
+          fragment.at('div').add_child(Nokogiri::XML::Text.new(label, fragment))
+        end
+      end
+
+      def formatter
+        @formatter ||= Rouge::Formatters::HTML.new
+      end
+
+      def format(lexer, source)
+        tokens = lexer.lex(source)
+        formatter.format(tokens)
+      end
+    end
+  end
+end

data/lib/mato/html_filters/task_list.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Mato
+  module HtmlFilters
+    class TaskList
+      CHECKED_MARK = "[x] "
+      UNCHECKED_MARK = "[ ] "
+
+      DEFAULT_TASK_LIST_CLASS = "task-list-item"
+      DEFAULT_CHECKBOX_CLASS = "task-list-item-checkbox"
+
+      def initialize(task_list_class: DEFAULT_TASK_LIST_CLASS, checkbox_class: DEFAULT_CHECKBOX_CLASS)
+        @task_list_class = task_list_class
+        @checkbox_class = checkbox_class
+      end
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        doc.search("li").each do |li|
+          weave(li)
+        end
+      end
+
+      # @param [Nokogiri::XML::Node] li
+      def weave(li)
+        text_node = li.xpath('.//text()').first
+        checked = has_checked_mark?(text_node)
+        unchecked = has_unchecked_mark?(text_node)
+
+        return unless checked || unchecked
+
+        li["class"] = @task_list_class
+
+        text_node.content = trim_mark(text_node.content, checked)
+        checkbox = build_checkbox_node(checked)
+        text_node.add_previous_sibling(checkbox)
+      end
+
+      def has_checked_mark?(text_node)
+        text_node&.content&.start_with?(CHECKED_MARK)
+      end
+
+      def has_unchecked_mark?(text_node)
+        text_node&.content&.start_with?(UNCHECKED_MARK)
+      end
+
+      def trim_mark(content, checked)
+        if checked
+          content.sub(CHECKED_MARK, '')
+        else
+          content.sub(UNCHECKED_MARK, '')
+        end
+      end
+
+      def build_checkbox_node(checked)
+        Nokogiri::HTML.fragment('<input type="checkbox"/>').tap do |fragment|
+          checkbox = fragment.children.first
+          checkbox["class"] = @checkbox_class
+          checkbox["disabled"] = 'disabled'
+          checkbox["checked"] = 'checked' if checked
+        end
+      end
+    end
+  end
+end

data/lib/mato/html_filters/token_link.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative '../concerns/html_node_checkable'
+
+module Mato
+  module HtmlFilters
+    class TokenLink
+      include Concerns::HtmlNodeCheckable
+
+      # @return [Regexp]
+      attr_reader :pattern
+
+      # @return [Proc]
+      attr_reader :builder
+
+      # @param [Regexp] pattern
+      # @param [Procc] builder link builder that takes
+      def initialize(pattern, &builder)
+        @pattern = pattern
+        @builder = builder
+      end
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        doc.xpath('.//text()').each do |text_node|
+          next if has_ancestor?(text_node, 'a', 'code')
+
+          text_node.replace(text_node.content.gsub(pattern, &builder))
+        end
+      end
+    end
+  end
+end

data/lib/mato/processor.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative('./document')
+
+require 'nokogiri'
+require 'commonmarker'
+
+module Mato
+  class Processor
+    # @return [Mato::Config]
+    attr_reader :config
+
+    def initialize(config)
+      @config = config
+    end
+
+    # @param [String] input
+    # @return [Mato::Document]
+    def process(input)
+      text = input.dup
+
+      config.text_filters.each do |filter|
+        filter.call(text)
+      end
+
+      markdown_node = parse_markdown(text)
+
+      config.markdown_filters.each do |filter|
+        filter.call(markdown_node)
+      end
+
+      html = render_to_html(markdown_node)
+      doc = parse_html(html)
+
+      config.html_filters.each do |filter|
+        filter.call(doc)
+      end
+
+      config.document_factory.new(doc.freeze)
+    end
+
+    # @param [String] text
+    # @return [CommonMarker::Node]
+    def parse_markdown(text)
+      config.markdown_parser.render_doc(text, config.markdown_parse_options, config.markdown_extensions)
+    end
+
+    # @param [CommonMarker::Node] markdown_node
+    # @return [String]
+    def render_to_html(markdown_node)
+      markdown_node.to_html(config.markdown_render_options)
+    end
+
+    # @param [String] html
+    # @return [Nokogiri::HTML::DocumentFragment]
+    def parse_html(html)
+      config.html_parser.parse(html)
+    end
+  end
+end

data/lib/mato/renderers/html_renderer.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Mato
+  module Renderers
+    class HtmlRenderer
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      # @return [String]
+      def call(doc)
+        doc.to_html
+      end
+    end
+  end
+end

data/lib/mato/renderers/html_toc_renderer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative '../anchor_builder'
+
+module Mato
+  module Renderers
+    class HtmlTocRenderer
+
+      H_SELECTOR = %w(h1 h2 h3 h4 h5 h6).join(',')
+
+      # @param [Nokogiri::HTML::DocumentFragment] doc
+      def call(doc)
+        s = +''
+
+        stack = [0]
+
+        doc.css(H_SELECTOR).each do |hx|
+          h_level = level(hx)
+          if h_level > stack.last
+            stack.push(h_level)
+            s << %{<ul>\n}
+          elsif h_level < stack.last
+            while stack.last > h_level
+              s << %{</li></ul>\n}
+              stack.pop
+            end
+          end
+
+          first_child = hx.child
+          if anchor?(first_child)
+            s << %{<li><a href="##{first_child['id']}">}
+
+            child = first_child.next_sibling
+            while child
+              s << child.to_html
+              child = child.next_sibling
+            end
+
+            s << %{</a>}
+          else
+            s << %{<li>#{hx.children}}
+          end
+        end
+
+        while stack.last != 0
+          stack.pop
+          s << %{</li></ul>\n}
+        end
+
+        s
+      end
+
+      private
+
+      # @param [Nokogiri::XML::Node] node
+      def level(node)
+        /\d+/.match(node.name)[0].to_i
+      end
+
+      # @param [Nokogiri::XML::Node] node
+      def anchor?(node)
+        node.name == 'a' && node['class'] == AnchorBuilder::CSS_CLASS_NAME
+      end
+    end
+  end
+end

data/lib/mato/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mato
+  VERSION = "0.1.0"
+end

data/mato.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative './lib/mato/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mato"
+  spec.version       = Mato::VERSION
+  spec.authors       = ["FUJI Goro"]
+  spec.email         = ["goro-fuj@bitjoureny.com"]
+
+  spec.summary       = 'MArkdown TOolkit'
+  spec.description   = 'An extensible markdown processing toolkit based on CommonMark'
+  spec.homepage      = "https://github.com/bitjourney/mato"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = ">= 2.3"
+
+  spec.add_runtime_dependency "nokogiri", ">= 1.6"
+  spec.add_runtime_dependency "commonmarker", ">= 0.14"
+
+  # spec.add_optional_dependency "sanitize", ">= 3.0"
+  # spec.add_optional_dependency "rouge", ">= 2.0"
+
+  spec.add_development_dependency "bundler", ">= 1.14"
+  spec.add_development_dependency "rake", ">= 10.0"
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: mato
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- FUJI Goro
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: commonmarker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: An extensible markdown processing toolkit based on CommonMark
+email:
+- goro-fuj@bitjoureny.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example/hello.rb
+- example/toc.rb
+- lib/mato.rb
+- lib/mato/anchor_builder.rb
+- lib/mato/concerns/html_node_checkable.rb
+- lib/mato/config.rb
+- lib/mato/document.rb
+- lib/mato/html_filters/mention_link.rb
+- lib/mato/html_filters/sanitization.rb
+- lib/mato/html_filters/section_anchor.rb
+- lib/mato/html_filters/syntax_highlight.rb
+- lib/mato/html_filters/task_list.rb
+- lib/mato/html_filters/token_link.rb
+- lib/mato/processor.rb
+- lib/mato/renderers/html_renderer.rb
+- lib/mato/renderers/html_toc_renderer.rb
+- lib/mato/version.rb
+- mato.gemspec
+homepage: https://github.com/bitjourney/mato
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: MArkdown TOolkit
+test_files: []