qiita_marker 0.23.2.0 → 0.23.2.3

data/ext/qiita_marker/qfm_mention_no_emphasis.c

@@ -0,0 +1,37 @@
+#include "cmark-gfm.h"
+#include "cmark_ctype.h"
+#include "config.h"
+
+static bool is_wordchar(char c) {
+  return cmark_isalnum(c) || c == '_' || c == '-';
+}
+
+bool is_part_of_mention(unsigned char *data, bufsize_t offset) {
+  int i;
+  int lookbehind_limit = (int)-offset;
+  char character;
+
+  for (i = 0; i >= lookbehind_limit; i--) {
+    character = data[i];
+
+    if (is_wordchar(character)) {
+      // Continue lookbehind.
+    } else if (character == '@') {
+      if (i == offset) {
+        // The "@" is at beginning of the text. (e.g. "@foo")
+        return true;
+      } else {
+        // Check if the previous character of the "@" is alphanumeric or not.
+        //   " @foo" and "あ@foo" are mentions.
+        //   "a@foo" is not a mention.
+        char prev_character = data[i - 1];
+        return !cmark_isalnum(prev_character);
+      }
+    } else {
+      // Found non-mention character, so this is not a mention.
+      return false;
+    }
+  }
+
+  return false;
+}

data/ext/qiita_marker/qfm_mention_no_emphasis.h

@@ -0,0 +1,8 @@
+#ifndef QFM_MENTION_NO_EMPHASIS
+#define QFM_MENTION_NO_EMPHASIS
+
+#include "cmark-gfm.h"
+
+bool is_part_of_mention(unsigned char *data, bufsize_t offset);
+
+#endif

data/ext/qiita_marker/qfm_scanners.c

@@ -0,0 +1,291 @@
+/* Generated by re2c 1.3 */
+
+#include "qfm_scanners.h"
+#include <stdlib.h>
+
+bufsize_t _qfm_scan_at(bufsize_t (*scanner)(const unsigned char *),
+                       unsigned char *ptr, int len, bufsize_t offset) {
+  bufsize_t res;
+
+  if (ptr == NULL || offset >= len) {
+    return 0;
+  } else {
+    unsigned char lim = ptr[len];
+
+    ptr[len] = '\0';
+    res = scanner(ptr + offset);
+    ptr[len] = lim;
+  }
+
+  return res;
+}
+
+// Scan an opening qfm_custom_block fence.
+bufsize_t _scan_open_qfm_custom_block_fence(const unsigned char *p) {
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+
+  {
+    unsigned char yych;
+    static const unsigned char yybm[] = {
+        0,   128, 128, 128, 128, 128, 128, 128, 128, 128, 0,   128, 128, 0,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 64,  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+        128, 128, 0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,
+        0,   0,   0,   0,
+    };
+    yych = *p;
+    if (yych == ':')
+      goto yy4;
+    ++p;
+  yy3 : { return 0; }
+  yy4:
+    yych = *(marker = ++p);
+    if (yych != ':')
+      goto yy3;
+    yych = *++p;
+    if (yybm[0 + yych] & 64) {
+      goto yy7;
+    }
+  yy6:
+    p = marker;
+    goto yy3;
+  yy7:
+    yych = *++p;
+    if (yybm[0 + yych] & 64) {
+      goto yy7;
+    }
+    if (yych <= 0xDF) {
+      if (yych <= '\f') {
+        if (yych <= 0x00)
+          goto yy6;
+        if (yych == '\n') {
+          marker = p;
+          goto yy11;
+        }
+        marker = p;
+      } else {
+        if (yych <= '\r') {
+          marker = p;
+          goto yy11;
+        }
+        if (yych <= 0x7F) {
+          marker = p;
+          goto yy9;
+        }
+        if (yych <= 0xC1)
+          goto yy6;
+        marker = p;
+        goto yy13;
+      }
+    } else {
+      if (yych <= 0xEF) {
+        if (yych <= 0xE0) {
+          marker = p;
+          goto yy14;
+        }
+        if (yych == 0xED) {
+          marker = p;
+          goto yy16;
+        }
+        marker = p;
+        goto yy15;
+      } else {
+        if (yych <= 0xF0) {
+          marker = p;
+          goto yy17;
+        }
+        if (yych <= 0xF3) {
+          marker = p;
+          goto yy18;
+        }
+        if (yych <= 0xF4) {
+          marker = p;
+          goto yy19;
+        }
+        goto yy6;
+      }
+    }
+  yy9:
+    yych = *++p;
+    if (yybm[0 + yych] & 128) {
+      goto yy9;
+    }
+    if (yych <= 0xEC) {
+      if (yych <= 0xC1) {
+        if (yych <= 0x00)
+          goto yy6;
+        if (yych >= 0x0E)
+          goto yy6;
+      } else {
+        if (yych <= 0xDF)
+          goto yy13;
+        if (yych <= 0xE0)
+          goto yy14;
+        goto yy15;
+      }
+    } else {
+      if (yych <= 0xF0) {
+        if (yych <= 0xED)
+          goto yy16;
+        if (yych <= 0xEF)
+          goto yy15;
+        goto yy17;
+      } else {
+        if (yych <= 0xF3)
+          goto yy18;
+        if (yych <= 0xF4)
+          goto yy19;
+        goto yy6;
+      }
+    }
+  yy11:
+    ++p;
+    p = marker;
+    { return (bufsize_t)(p - start); }
+  yy13:
+    yych = *++p;
+    if (yych <= 0x7F)
+      goto yy6;
+    if (yych <= 0xBF)
+      goto yy9;
+    goto yy6;
+  yy14:
+    yych = *++p;
+    if (yych <= 0x9F)
+      goto yy6;
+    if (yych <= 0xBF)
+      goto yy13;
+    goto yy6;
+  yy15:
+    yych = *++p;
+    if (yych <= 0x7F)
+      goto yy6;
+    if (yych <= 0xBF)
+      goto yy13;
+    goto yy6;
+  yy16:
+    yych = *++p;
+    if (yych <= 0x7F)
+      goto yy6;
+    if (yych <= 0x9F)
+      goto yy13;
+    goto yy6;
+  yy17:
+    yych = *++p;
+    if (yych <= 0x8F)
+      goto yy6;
+    if (yych <= 0xBF)
+      goto yy15;
+    goto yy6;
+  yy18:
+    yych = *++p;
+    if (yych <= 0x7F)
+      goto yy6;
+    if (yych <= 0xBF)
+      goto yy15;
+    goto yy6;
+  yy19:
+    yych = *++p;
+    if (yych <= 0x7F)
+      goto yy6;
+    if (yych <= 0x8F)
+      goto yy15;
+    goto yy6;
+  }
+}
+
+// Scan a closing qfm_custom_block fence with length at least len.
+bufsize_t _scan_close_qfm_custom_block_fence(const unsigned char *p) {
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+
+  {
+    unsigned char yych;
+    static const unsigned char yybm[] = {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   128, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0,   0,   0, 0, 0,
+    };
+    yych = *p;
+    if (yych == ':')
+      goto yy24;
+    ++p;
+  yy23 : { return 0; }
+  yy24:
+    yych = *(marker = ++p);
+    if (yych != ':')
+      goto yy23;
+    yych = *++p;
+    if (yybm[0 + yych] & 64) {
+      goto yy27;
+    }
+  yy26:
+    p = marker;
+    goto yy23;
+  yy27:
+    yych = *++p;
+    if (yybm[0 + yych] & 64) {
+      goto yy27;
+    }
+    if (yych <= '\f') {
+      if (yych <= 0x08)
+        goto yy26;
+      if (yych <= '\t') {
+        marker = p;
+        goto yy29;
+      }
+      if (yych <= '\n') {
+        marker = p;
+        goto yy31;
+      }
+      goto yy26;
+    } else {
+      if (yych <= '\r') {
+        marker = p;
+        goto yy31;
+      }
+      if (yych != ' ')
+        goto yy26;
+      marker = p;
+    }
+  yy29:
+    yych = *++p;
+    if (yybm[0 + yych] & 128) {
+      goto yy29;
+    }
+    if (yych <= 0x08)
+      goto yy26;
+    if (yych <= '\n')
+      goto yy31;
+    if (yych != '\r')
+      goto yy26;
+  yy31:
+    ++p;
+    p = marker;
+    { return (bufsize_t)(p - start); }
+  }
+}

data/ext/qiita_marker/qfm_scanners.h

@@ -0,0 +1,20 @@
+#include "chunk.h"
+#include "cmark-gfm.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bufsize_t _qfm_scan_at(bufsize_t (*scanner)(const unsigned char *),
+                       unsigned char *ptr, int len, bufsize_t offset);
+bufsize_t _scan_open_qfm_custom_block_fence(const unsigned char *p);
+bufsize_t _scan_close_qfm_custom_block_fence(const unsigned char *p);
+
+#define scan_open_qfm_custom_block_fence(c, l, n)                              \
+  _qfm_scan_at(&_scan_open_qfm_custom_block_fence, c, l, n)
+#define scan_close_qfm_custom_block_fence(c, l, n)                             \
+  _qfm_scan_at(&_scan_close_qfm_custom_block_fence, c, l, n)
+
+#ifdef __cplusplus
+}
+#endif

data/ext/qiita_marker/qfm_scanners.re

@@ -0,0 +1,50 @@
+/*!re2c re2c:flags:no-debug-info = 1; */
+/*!re2c re2c:indent:string = '  '; */
+
+#include "qfm_scanners.h"
+#include <stdlib.h>
+
+bufsize_t _qfm_scan_at(bufsize_t (*scanner)(const unsigned char *),
+                       unsigned char *ptr, int len, bufsize_t offset) {
+  bufsize_t res;
+
+  if (ptr == NULL || offset >= len) {
+    return 0;
+  } else {
+    unsigned char lim = ptr[len];
+
+    ptr[len] = '\0';
+    res = scanner(ptr + offset);
+    ptr[len] = lim;
+  }
+
+  return res;
+}
+
+/*!re2c
+  re2c:define:YYCTYPE  = "unsigned char";
+  re2c:define:YYCURSOR = p;
+  re2c:define:YYMARKER = marker;
+  re2c:define:YYCTXMARKER = marker;
+  re2c:yyfill:enable = 0;
+*/
+
+// Scan an opening qfm_custom_block fence.
+bufsize_t _scan_open_qfm_custom_block_fence(const unsigned char *p) {
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+  /*!re2c
+    [:]{3,} / [^:\r\n\x00]*[\r\n] { return (bufsize_t)(p - start); }
+    * { return 0; }
+  */
+}
+
+// Scan a closing qfm_custom_block fence with length at least len.
+bufsize_t _scan_close_qfm_custom_block_fence(const unsigned char *p) {
+  const unsigned char *marker = NULL;
+  const unsigned char *start = p;
+  /*!re2c
+    [:]{3,} / [ \t]*[\r\n] { return (bufsize_t)(p - start); }
+    * { return 0; }
+  */
+}

data/lib/qiita_marker/config.rb

@@ -13,7 +13,9 @@ module QiitaMarker
         SMART: (1 << 10),
         LIBERAL_HTML_TAG: (1 << 12),
         FOOTNOTES: (1 << 13),
-        STRIKETHROUGH_DOUBLE_TILDE: (1 << 14)
+        STRIKETHROUGH_DOUBLE_TILDE: (1 << 14),
+        MENTION_NO_EMPHASIS: (1 << 26),
+        AUTOLINK_CLASS_NAME: (1 << 27)
       }.freeze,
       render: {
         DEFAULT: 0,
@@ -28,7 +30,10 @@ module QiitaMarker
         FOOTNOTES: (1 << 13),
         STRIKETHROUGH_DOUBLE_TILDE: (1 << 14),
         TABLE_PREFER_STYLE_ATTRIBUTES: (1 << 15),
-        FULL_INFO_STRING: (1 << 16)
+        FULL_INFO_STRING: (1 << 16),
+        CODE_DATA_METADATA: (1 << 25),
+        MENTION_NO_EMPHASIS: (1 << 26),
+        AUTOLINK_CLASS_NAME: (1 << 27)
       }.freeze,
       format: %i[html xml commonmark plaintext].freeze
     }.freeze

data/lib/qiita_marker/renderer/html_renderer.rb

@@ -94,6 +94,10 @@ module QiitaMarker
           out("<pre#{sourcepos(node)}")
           out(' lang="', node.fence_info.split(/\s+/)[0], '"') if node.fence_info && !node.fence_info.empty?
           out('><code>')
+        elsif option_enabled?(:CODE_DATA_METADATA)
+          out("<pre#{sourcepos(node)}><code")
+          out(' data-metadata="', node.fence_info, '"') if node.fence_info && !node.fence_info.empty?
+          out('>')
         else
           out("<pre#{sourcepos(node)}><code")
           if node.fence_info && !node.fence_info.empty?

data/lib/qiita_marker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module QiitaMarker
-  VERSION = '0.23.2.0'
+  VERSION = '0.23.2.3'
 end

data/qiita_marker.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |s|
   s.version = QiitaMarker::VERSION
   s.summary = 'Qiita Marker is a Ruby library for Markdown processing, based on CommonMarker.'
   s.description = 'A Ruby library that is the core module of the Qiita-specified markdown processor.'
-  s.authors = ['Increments Inc.']
+  s.authors = ['Qiita Inc.']
   s.homepage = 'https://github.com/increments/qiita_marker'
   s.license = 'MIT'
 

data/test/test_qfm_autolink_class_name.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require('test_helper')
+
+describe 'TestQfmAutolinkClassName' do
+  let(:options) { %i[AUTOLINK_CLASS_NAME] }
+  let(:extensions) { %i[autolink] }
+  let(:text) do
+    <<~MD
+      https://example.com
+      <https://example.com>
+      [Example](https://example.com)
+      test@example.com
+    MD
+  end
+  let(:doc) { QiitaMarker.render_doc(text, options, extensions) }
+  let(:expected) do
+    <<~HTML
+      <p><a href="https://example.com" class="autolink">https://example.com</a>
+      <a href="https://example.com">https://example.com</a>
+      <a href="https://example.com">Example</a>
+      <a href="mailto:test@example.com" class="autolink">test@example.com</a></p>
+    HTML
+  end
+  let(:rendered_html) { doc.to_html(options, extensions) }
+
+  it "appends class name to extension's autolinks" do
+    assert_equal(expected, rendered_html)
+  end
+
+  describe 'without AUTOLINK_CLASS_NAME option' do
+    let(:options) { %i[DEFAULT] }
+    let(:expected) do
+      <<~HTML
+        <p><a href="https://example.com">https://example.com</a>
+        <a href="https://example.com">https://example.com</a>
+        <a href="https://example.com">Example</a>
+        <a href="mailto:test@example.com">test@example.com</a></p>
+      HTML
+    end
+
+    it "does not append class name to extension's autolink" do
+      assert_equal(expected, rendered_html)
+    end
+  end
+
+  describe 'without autolink extension' do
+    let(:extensions) { %i[] }
+    let(:expected) do
+      <<~HTML
+        <p>https://example.com
+        <a href="https://example.com">https://example.com</a>
+        <a href="https://example.com">Example</a>
+        test@example.com</p>
+      HTML
+    end
+
+    it 'does not append class name' do
+      assert_equal(expected, rendered_html)
+    end
+  end
+end

data/test/test_qfm_code_data_metadata.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require('test_helper')
+
+class TestQfmCodeDataMetadata < Minitest::Test
+  def test_to_html
+    text = <<~MD
+      ```ruby:example main.rb
+      puts :foo
+      ```
+    MD
+    doc = render_doc(text)
+    expected = <<~HTML
+      <pre><code data-metadata="ruby:example main.rb">puts :foo
+      </code></pre>
+    HTML
+
+    assert_equal(expected, doc.to_html(:CODE_DATA_METADATA))
+    assert_equal(expected, QiitaMarker::HtmlRenderer.new(options: :CODE_DATA_METADATA).render(doc))
+  end
+
+  def test_with_character_reference
+    text = <<~MD
+      ```ruby:example&#x20;main.rb
+      puts :foo
+      ```
+    MD
+    doc = render_doc(text)
+    expected = <<~HTML
+      <pre><code data-metadata="ruby:example main.rb">puts :foo
+      </code></pre>
+    HTML
+
+    assert_equal(expected, doc.to_html(:CODE_DATA_METADATA))
+    assert_equal(expected, QiitaMarker::HtmlRenderer.new(options: :CODE_DATA_METADATA).render(doc))
+  end
+
+  def render_doc(markdown)
+    QiitaMarker.render_doc(markdown, :DEFAULT, [])
+  end
+end

data/test/test_qfm_custom_block.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require('test_helper')
+
+class TestQfmCustomBlock < Minitest::Test
+  def setup
+    text = <<~MD
+      :::foo bar
+      message
+
+      - list1
+      - list2
+
+      ```ruby
+      puts 'hello'
+      ```
+
+      <div>html block</div>
+      :::
+    MD
+    @doc = QiitaMarker.render_doc(text, %i[UNSAFE], %i[custom_block])
+    @expected = <<~HTML
+      <div data-type="customblock" data-metadata="foo bar">
+      <p>message</p>
+      <ul>
+      <li>list1</li>
+      <li>list2</li>
+      </ul>
+      <pre><code class="language-ruby">puts 'hello'
+      </code></pre>
+      <div>html block</div>
+      </div>
+    HTML
+  end
+
+  def test_to_html
+    assert_equal(@expected, @doc.to_html(%i[UNSAFE], %i[custom_block]))
+  end
+end

data/test/test_qfm_mention_no_emphasis.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TestQfmMentionNoEmphasis < Minitest::Test
+  describe 'with mention_no_emphasis option' do
+    [
+      ['@_username_',         false],
+      ['@__username__',       false],
+      ['@___username___',     false],
+      ['@user__name__',       false],
+      ['@some__user__name__', false],
+      [' @_username_',        false],
+      ['あ@_username_', false],
+      ['A@_username_',        true],
+      ['@*username*',         true],
+      ['_foo_',               true],
+      ['_',                   false],
+      ['_foo @username_',     false],
+      ['__foo @username__',   false],
+      ['___foo @username___', false]
+    ].each do |text, emphasize|
+      describe "with text #{text.inspect}" do
+        if emphasize
+          it 'emphasizes the text' do
+            QiitaMarker.render_html(text, :MENTION_NO_EMPHASIS).tap do |out|
+              assert_match(/(<em>|<strong>)/, out.chomp)
+            end
+          end
+        else
+          it 'does not emphasize the text' do
+            QiitaMarker.render_html(text, :MENTION_NO_EMPHASIS).tap do |out|
+              assert_match "<p>#{text.strip}</p>", out.chomp
+            end
+          end
+        end
+      end
+    end
+  end
+
+  describe 'without mention_no_emphasis option' do
+    describe 'with text "@_username_"' do
+      text = '@_username_'
+      it 'emphasizes the text' do
+        QiitaMarker.render_html(text, :DEFAULT, %i[]).tap do |out|
+          assert_match(/<em>/, out.chomp)
+        end
+      end
+    end
+
+    describe 'with text "_foo @username_"' do
+      text = '_foo @username_'
+      it 'emphasizes the text' do
+        QiitaMarker.render_html(text, :DEFAULT, %i[]).tap do |out|
+          assert_match(/<em>/, out.chomp)
+        end
+      end
+    end
+  end
+end