qiita_marker 0.23.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b4222f9db082f683cdabb0d9d42a255ab4b689fe19cdb178efe4feccea10439d
+  data.tar.gz: 49e29b6cee35cb99e550f65ed830e6cd83448c7b058c09c3383c97722df86b5c
+SHA512:
+  metadata.gz: c74fe17ddee587a42758795f9a9ff9bd5982b7c284a247615af864e6fdad8dd1b6708b55eadbec2b9eaf893e3d9eeb5e03c72d62da78598322b90d01adc8cc84
+  data.tar.gz: f8180555767d3aa6134f7dae895d06fdda9f4b655d41d6872b6234dbe073787debd4fd0b6956da261f149b28d821d95c6e0b947e7049307a92267205e24a5a9c

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright 2021 Increments Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Qiita Marker
+
+[![Build Status](https://github.com/increments/qiita-marker/actions/workflows/test.yml/badge.svg)](https://github.com/increments/qiita-marker/actions/workflows/test.yml)
+
+:warning: This library is still in the testing phase. As such, development may be halted.
+
+Qiita Marker is a Ruby library for Markdown processing, based on [CommonMarker](https://github.com/gjtorikian/commonmarker).
+It will be a core module of [Qiita Markdown](https://github.com/increments/qiita-markdown) gem and not intended for direct use. If you are looking for Qiita-specified markdown processor, use [Qiita Markdown](https://github.com/increments/qiita-markdown) gem.
+
+## Usage
+
+Please see [CommonMarker's Usage](https://github.com/gjtorikian/commonmarker#usage).
+
+## Contributing
+
+If you have suggestion or modification to this repository, please create an Issue or Pull Request.
+
+### How to develop
+
+```
+# Clone repository
+$ git clone git@github.com:increments/qiita_marker.git
+$ cd qiita_marker
+
+# Setup development environment
+$ ./script/bootstrap
+
+# Run test
+$ bundle exec rake test
+```
+
+#### with Docker
+
+```
+# Clone repository
+$ git clone git@github.com:increments/qiita_marker.git
+$ cd qiita_marker
+
+# Setup development environment
+$ docker compose build
+$ docker compose up -d
+$ docker compose run --rm app ./script/bootstrap
+
+# Run test
+$ docker compose run --rm rake test
+```
+
+## License
+
+Please see [LICENSE.txt](/LICENSE.txt).

data/Rakefile

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'rake/clean'
+require 'rake/extensiontask'
+require 'digest/md5'
+
+host_os = RbConfig::CONFIG['host_os']
+require 'devkit' if host_os == 'mingw32'
+
+task default: [:test]
+
+# Gem Spec
+gem_spec = Gem::Specification.load('qiita_marker.gemspec')
+
+# Ruby Extension
+Rake::ExtensionTask.new('qiita_marker', gem_spec) do |ext|
+  ext.lib_dir = File.join('lib', 'qiita_marker')
+end
+
+# Packaging
+require 'bundler/gem_tasks'
+
+# Testing
+require 'rake/testtask'
+
+Rake::TestTask.new('test:unit') do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/test_*.rb'
+  t.verbose = true
+  t.warning = false
+end
+
+task 'test:unit' => :compile
+
+desc 'Run unit and conformance tests'
+task test: %w[test:unit]
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop)
+
+desc 'Run benchmarks'
+task :benchmark do
+  if ENV['FETCH_PROGIT']
+    `rm -rf test/progit`
+    `git clone https://github.com/progit/progit.git test/progit`
+    langs = %w[ar az be ca cs de en eo es es-ni fa fi fr hi hu id it ja ko mk nl no-nb pl pt-br ro ru sr th tr uk vi zh zh-tw]
+    langs.each do |lang|
+      `cat test/progit/#{lang}/*/*.markdown >> test/benchinput.md`
+    end
+  end
+  $LOAD_PATH.unshift 'lib'
+  load 'test/benchmark.rb'
+end
+
+desc 'Match C style of cmark'
+task :format do
+  sh 'clang-format -style llvm -i ext/qiita_marker/*.c ext/qiita_marker/*.h'
+end
+
+# Documentation
+require 'rdoc/task'
+
+desc 'Generate API documentation'
+RDoc::Task.new do |rd|
+  rd.rdoc_dir = 'docs'
+  rd.main     = 'README.md'
+  rd.rdoc_files.include 'README.md', 'lib/**/*.rb', 'ext/qiita_marker/qiita_marker.c'
+
+  rd.options << '--markup tomdoc'
+  rd.options << '--inline-source'
+  rd.options << '--line-numbers'
+  rd.options << '--all'
+  rd.options << '--fileboxes'
+end
+
+desc 'Generate the documentation and run a web server'
+task serve: [:rdoc] do
+  require 'webrick'
+
+  puts 'Navigate to http://localhost:3000 to see the docs'
+
+  server = WEBrick::HTTPServer.new Port: 3000
+  server.mount '/', WEBrick::HTTPServlet::FileHandler, 'docs'
+  trap('INT') { server.stop }
+  server.start
+end
+
+desc 'Generate and publish docs to gh-pages'
+task publish: [:rdoc] do
+  require 'tmpdir'
+  require 'shellwords'
+
+  Dir.mktmpdir do |tmp|
+    system "mv docs/* #{tmp}"
+    system 'git checkout origin/gh-pages'
+    system 'rm -rf *'
+    system "mv #{tmp}/* ."
+    message = Shellwords.escape("Site updated at #{Time.now.utc}")
+    system 'git add .'
+    system "git commit -am #{message}"
+    system 'git push origin gh-pages --force'
+    system 'git checkout master'
+    system 'echo yolo'
+  end
+end
+
+# Custom tasks
+Dir.glob('tasks/*.rake').each do |f|
+  load f
+end

data/bin/qiita_marker

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'optparse'
+require 'ostruct'
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+require 'qiita_marker'
+
+root = File.expand_path('..', __dir__)
+$LOAD_PATH.unshift File.expand_path('lib', root)
+
+def parse_options
+  options = OpenStruct.new
+  extensions = QiitaMarker.extensions
+  parse_options = QiitaMarker::Config::OPTS.fetch(:parse)
+  render_options = QiitaMarker::Config::OPTS.fetch(:render)
+  format_options = QiitaMarker::Config::OPTS.fetch(:format)
+
+  options.active_extensions = []
+  options.active_parse_options = [:DEFAULT]
+  options.active_render_options = [:DEFAULT]
+  options.output_format = :html
+
+  option_parser = OptionParser.new do |opts|
+    opts.banner = 'Usage: qiita_marker [--html-renderer] [--extension=EXTENSION]'
+    opts.separator '                    [--to=FORMAT]'
+    opts.separator '                    [--parse-option=OPTION]'
+    opts.separator '                    [--render-option=OPTION]'
+    opts.separator '                    [FILE..]'
+    opts.separator ''
+    opts.separator 'Convert one or more CommonMark files to HTML and write to standard output.'
+    opts.separator 'If no FILE argument is provided, text will be read from STDIN.'
+    opts.separator ''
+
+    opts.on('--extension=EXTENSION', Array, 'Use EXTENSION for parsing and HTML output (unless --html-renderer is specified)') do |values|
+      values.each do |value|
+        if extensions.include?(value)
+          options.active_extensions << value.to_sym
+        else
+          abort("extension '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-h', '--help', 'Prints this help') do
+      puts opts
+      puts
+      puts "Available formats: #{format_options.join(', ')}"
+      puts "Available extentions: #{extensions.join(', ')}"
+      puts "Available parse options: #{parse_options.keys.join(', ')}"
+      puts "Available render options: #{render_options.keys.join(', ')}"
+      puts
+      puts 'See the README for more information on these.'
+      exit
+    end
+
+    opts.on('-tFORMAT', '--to=FORMAT', String, 'Specify output FORMAT') do |value|
+      value = value.to_sym
+      if format_options.include?(value)
+        options.output_format = value
+      else
+        abort("format '#{value}' not found")
+      end
+    end
+
+    opts.on('--html-renderer', 'Use the HtmlRenderer renderer rather than the native C renderer (only valid when format is html)') do
+      options.renderer = true
+    end
+
+    opts.on('--parse-option=OPTION', Array, 'OPTION passed during parsing') do |values|
+      values.each do |value|
+        if parse_options.key?(value.to_sym)
+          options.active_parse_options << value.to_sym
+        else
+          abort("parse-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('--render-option=OPTION', Array, 'OPTION passed during rendering') do |values|
+      values.each do |value|
+        if render_options.key?(value.to_sym)
+          options.active_render_options << value.to_sym
+        else
+          abort("render-option '#{value}' not found")
+        end
+      end
+    end
+
+    opts.on('-v', '--version', 'Version information') do
+      puts "qiita_marker #{QiitaMarker::VERSION}"
+      exit
+    end
+  end
+
+  option_parser.parse!
+
+  options
+end
+
+options = parse_options
+
+abort("format '#{options.output_format}' does not support using the HtmlRenderer renderer") if
+  options.renderer && options.output_format != :html
+
+doc = QiitaMarker.render_doc(ARGF.read, options.active_parse_options, options.active_extensions)
+
+case options.output_format
+when :html
+  if options.renderer
+    renderer = QiitaMarker::HtmlRenderer.new(options: options.active_render_options, extensions: options.active_extensions)
+    $stdout.write(renderer.render(doc))
+  else
+    $stdout.write(doc.to_html(options.active_render_options, options.active_extensions))
+  end
+when :xml
+  $stdout.write(doc.to_xml(options.active_render_options))
+when :commonmark
+  $stdout.write(doc.to_commonmark(options.active_render_options))
+when :plaintext
+  $stdout.write(doc.to_plaintext(options.active_render_options))
+end

data/ext/qiita_marker/arena.c

@@ -0,0 +1,103 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include "cmark-gfm.h"
+#include "cmark-gfm-extension_api.h"
+
+static struct arena_chunk {
+  size_t sz, used;
+  uint8_t push_point;
+  void *ptr;
+  struct arena_chunk *prev;
+} *A = NULL;
+
+static struct arena_chunk *alloc_arena_chunk(size_t sz, struct arena_chunk *prev) {
+  struct arena_chunk *c = (struct arena_chunk *)calloc(1, sizeof(*c));
+  if (!c)
+    abort();
+  c->sz = sz;
+  c->ptr = calloc(1, sz);
+  if (!c->ptr)
+    abort();
+  c->prev = prev;
+  return c;
+}
+
+void cmark_arena_push(void) {
+  if (!A)
+    return;
+  A->push_point = 1;
+  A = alloc_arena_chunk(10240, A);
+}
+
+int cmark_arena_pop(void) {
+  if (!A)
+    return 0;
+  while (A && !A->push_point) {
+    free(A->ptr);
+    struct arena_chunk *n = A->prev;
+    free(A);
+    A = n;
+  }
+  if (A)
+    A->push_point = 0;
+  return 1;
+}
+
+static void init_arena(void) {
+  A = alloc_arena_chunk(4 * 1048576, NULL);
+}
+
+void cmark_arena_reset(void) {
+  while (A) {
+    free(A->ptr);
+    struct arena_chunk *n = A->prev;
+    free(A);
+    A = n;
+  }
+}
+
+static void *arena_calloc(size_t nmem, size_t size) {
+  if (!A)
+    init_arena();
+
+  size_t sz = nmem * size + sizeof(size_t);
+
+  // Round allocation sizes to largest integer size to
+  // ensure returned memory is correctly aligned
+  const size_t align = sizeof(size_t) - 1;
+  sz = (sz + align) & ~align;
+
+  if (sz > A->sz) {
+    A->prev = alloc_arena_chunk(sz, A->prev);
+    return (uint8_t *) A->prev->ptr + sizeof(size_t);
+  }
+  if (sz > A->sz - A->used) {
+    A = alloc_arena_chunk(A->sz + A->sz / 2, A);
+  }
+  void *ptr = (uint8_t *) A->ptr + A->used;
+  A->used += sz;
+  *((size_t *) ptr) = sz - sizeof(size_t);
+  return (uint8_t *) ptr + sizeof(size_t);
+}
+
+static void *arena_realloc(void *ptr, size_t size) {
+  if (!A)
+    init_arena();
+
+  void *new_ptr = arena_calloc(1, size);
+  if (ptr)
+    memcpy(new_ptr, ptr, ((size_t *) ptr)[-1]);
+  return new_ptr;
+}
+
+static void arena_free(void *ptr) {
+  (void) ptr;
+  /* no-op */
+}
+
+cmark_mem CMARK_ARENA_MEM_ALLOCATOR = {arena_calloc, arena_realloc, arena_free};
+
+cmark_mem *cmark_get_arena_mem_allocator() {
+  return &CMARK_ARENA_MEM_ALLOCATOR;
+}