redcarpet 2.0.1 → 2.1.0

data/README.markdown

@@ -66,7 +66,7 @@ settings, and reused between parses.
                 :tables - parse tables, PHP-Markdown style
 
                 :fenced_code_blocks - parse fenced code blocks, PHP-Markdown
-                    style .Blocks delimited with 3 or more `~` or backticks
+                    style. Blocks delimited with 3 or more `~` or backticks
                     will be considered as code, without the need to be
                     indented. An optional language name may be added at the
                     end of the opening fence for the code block

data/bin/redcarpet

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
-# Usage: redcarpet [<file>...]
+# Usage: redcarpet [--parse-<extension>...] [--render-<extension>] [<file>...]
 # Convert one or more Markdown files to HTML and write to standard output. With
 # no <file> or when <file> is '-', read Markdown source text from standard input.
+# With <extension>s, perform additional Markdown processing before writing output.
 if ARGV.include?('--help')
   File.read(__FILE__).split("\n").grep(/^# /).each do |line|
     puts line[2..-1]
@@ -9,5 +10,25 @@ if ARGV.include?('--help')
   exit 0
 end
 
+root = File.expand_path('../../', __FILE__)
+$:.unshift File.expand_path('lib', root)
+
 require 'redcarpet'
-STDOUT.write(Redcarpet::Markdown.new(Redcarpet::Render::HTML).render(ARGF.read))
+
+render_extensions = {}
+parse_extensions = {}
+
+ARGV.delete_if do |arg|
+  if arg =~ /^--render-([\w-]+)$/
+    arg = $1.gsub('-', '_')
+    render_extensions[arg.to_sym] = true
+  elsif arg =~ /^--parse-([\w-]+)$/
+    arg = $1.gsub('-', '_')
+    parse_extensions[arg.to_sym] = true
+  else
+    false
+  end
+end
+
+render = Redcarpet::Render::HTML.new(render_extensions)
+STDOUT.write(Redcarpet::Markdown.new(render, parse_extensions).render(ARGF.read))

data/ext/redcarpet/markdown.c

@@ -2361,6 +2361,7 @@ void
 sd_markdown_render(struct buf *ob, const uint8_t *document, size_t doc_size, struct sd_markdown *md)
 {
 #define MARKDOWN_GROW(x) ((x) + ((x) >> 1))
+	static const char UTF8_BOM[] = {0xEF, 0xBB, 0xBF};
 
 	struct buf *text;
 	size_t beg, end;
@@ -2377,6 +2378,12 @@ sd_markdown_render(struct buf *ob, const uint8_t *document, size_t doc_size, str
 
 	/* first pass: looking for references, copying everything else */
 	beg = 0;
+
+	/* Skip a possible UTF-8 BOM, even though the Unicode standard
+	 * discourages having these in UTF-8 documents */
+	if (doc_size >= 3 && memcmp(document, UTF8_BOM, 3) == 0)
+		beg += 3;
+
 	while (beg < doc_size) /* iterating over lines */
 		if (is_ref(document, beg, doc_size, &end, md->refs))
 			beg = end;

data/ext/redcarpet/rc_markdown.c

@@ -24,6 +24,8 @@ static void rb_redcarpet_md_flags(VALUE hash, unsigned int *enabled_extensions_p
 {
 	unsigned int extensions = 0;
 
+	Check_Type(hash, T_HASH);
+
 	/**
 	 * Markdown extensions -- all disabled by default
 	 */
@@ -89,18 +91,6 @@ static VALUE rb_redcarpet_md__new(int argc, VALUE *argv, VALUE klass)
 	return rb_markdown;
 }
 
-static void check_utf8_encoding(VALUE str)
-{
-#ifdef HAVE_RUBY_ENCODING_H
-	rb_encoding *enc = rb_enc_get(str);
-	if (enc != rb_utf8_encoding() && enc != rb_usascii_encoding()) {
-		rb_raise(rb_eTypeError,
-			"Input must be UTF-8 or US-ASCII, %s given", rb_enc_name(enc));
-	}
-#endif
-}
-
-
 static VALUE rb_redcarpet_md_render(VALUE self, VALUE text)
 {
 	VALUE rb_rndr;
@@ -108,7 +98,6 @@ static VALUE rb_redcarpet_md_render(VALUE self, VALUE text)
 	struct sd_markdown *markdown;
 
 	Check_Type(text, T_STRING);
-	check_utf8_encoding(text);
 
 	rb_rndr = rb_iv_get(self, "@renderer");
 	Data_Get_Struct(self, struct sd_markdown, markdown);
@@ -116,6 +105,14 @@ static VALUE rb_redcarpet_md_render(VALUE self, VALUE text)
 	if (rb_respond_to(rb_rndr, rb_intern("preprocess")))
 		text = rb_funcall(rb_rndr, rb_intern("preprocess"), 1, text);
 
+#ifdef HAVE_RUBY_ENCODING_H
+	{
+		struct rb_redcarpet_rndr *renderer;
+		Data_Get_Struct(rb_rndr, struct rb_redcarpet_rndr, renderer);
+		renderer->options.active_enc = rb_enc_get(text);
+	}
+#endif
+
 	/* initialize buffers */
 	output_buf = bufnew(128);
 
@@ -127,7 +124,7 @@ static VALUE rb_redcarpet_md_render(VALUE self, VALUE text)
 		markdown);
 
 	/* build the Ruby string */
-	text = redcarpet_str_new(output_buf->data, output_buf->size);
+	text = redcarpet_str_new(output_buf->data, output_buf->size, rb_enc_get(text));
 
 	bufrelease(output_buf);
 

data/ext/redcarpet/rc_render.c

@@ -40,13 +40,11 @@ VALUE rb_cRenderHTML;
 VALUE rb_cRenderHTML_TOC;
 VALUE rb_mSmartyPants;
 
-static inline VALUE
-buf2str(const struct buf *text)
-{
-	if (!text || !text->size) return Qnil;
-	return redcarpet_str_new(text->data, text->size);
-}
-
+#ifdef HAVE_RUBY_ENCODING_H
+#define buf2str(t) ((t) ? redcarpet_str_new((t)->data, (t)->size, opt->active_enc) : Qnil)
+#else
+#define buf2str(t) ((t) ? redcarpet_str_new((t)->data, (t)->size, NULL) : Qnil)
+#endif
 
 static void
 rndr_blockcode(struct buf *ob, const struct buf *text, const struct buf *lang, void *opaque)
@@ -413,7 +411,7 @@ static VALUE rb_redcarpet_smartypants_render(VALUE self, VALUE text)
 	output_buf = bufnew(128);
 
 	sdhtml_smartypants(output_buf, RSTRING_PTR(text), RSTRING_LEN(text));
-	result = redcarpet_str_new(output_buf->data, output_buf->size);
+	result = redcarpet_str_new(output_buf->data, output_buf->size, rb_enc_get(text));
 
 	bufrelease(output_buf);
 	return result;

data/ext/redcarpet/redcarpet.h

@@ -7,9 +7,9 @@
 
 #ifdef HAVE_RUBY_ENCODING_H
 #	include <ruby/encoding.h>
-#	define redcarpet_str_new(data, size) rb_enc_str_new(data, size, rb_utf8_encoding())
+#	define redcarpet_str_new(data, size, enc) rb_enc_str_new(data, size, enc)
 #else
-#	define redcarpet_str_new(data, size) rb_str_new(data, size)
+#	define redcarpet_str_new(data, size, enc) rb_str_new(data, size)
 #endif
 
 #include "markdown.h"
@@ -23,6 +23,9 @@ struct redcarpet_renderopt {
 	struct html_renderopt html;
 	VALUE self;
 	VALUE base_class;
+#ifdef HAVE_RUBY_ENCODING_H
+	rb_encoding *active_enc;
+#endif
 };
 
 struct rb_redcarpet_rndr {

data/lib/redcarpet.rb

@@ -1,7 +1,7 @@
 require 'redcarpet.so'
 
 module Redcarpet
-  VERSION = '2.0.1'
+  VERSION = '2.1.0'
 
   class Markdown
     attr_reader :renderer
@@ -54,20 +54,71 @@ module Redcarpet
 end
 
 # Compatibility class;
-# Creates a instance of Redcarpet with the RedCloth
-# API. This instance has no extensions enabled whatsoever,
-# and no accessors to change this. 100% pure, standard
-# Markdown.
+# Creates an instance of Redcarpet with the RedCloth API.
 class RedcarpetCompat
   attr_accessor :text
 
-  def initialize(text, *_dummy)
+  def initialize(text, *exts)
+    exts_hash, render_hash = *parse_extensions_and_renderer_options(exts)
     @text = text
-    @markdown = Redcarpet::Markdown.new(Redcarpet::Render::HTML)
+    renderer = Redcarpet::Render::HTML.new(render_hash)
+    @markdown = Redcarpet::Markdown.new(renderer, exts_hash)
   end
 
   def to_html(*_dummy)
     @markdown.render(@text)
   end
+  
+  private
+  
+  EXTENSION_MAP = {
+    # old name => new name
+    :autolink => :autolink,
+    :fenced_code => :fenced_code_blocks,
+    :filter_html => :filter_html,
+    :hard_wrap => :hard_wrap,
+    :lax_htmlblock => :lax_html_blocks,
+    :no_image => :no_images,
+    :no_intraemphasis => :no_intra_emphasis,
+    :no_links => :no_links,
+    :filter_styles => :no_styles,
+    :safelink => :safe_links_only,
+    :space_header => :space_after_headers,
+    :strikethrough => :strikethrough,
+    :tables => :tables,
+    :with_toc_data => :generate_toc,
+    :xhtml => :xhtml,
+    # old names with no new mapping
+    :gh_blockcode => nil,
+    :no_tables => nil,
+    :smart => nil,
+    :strict => nil
+  }
+  
+  RENDERER_OPTIONS = [:filter_html, :no_images, :no_links, :no_styles, 
+    :safe_links_only, :with_toc_data, :hard_wrap, :xhtml]
+  
+  def rename_extensions(exts)
+    exts.map do |old_name|
+      if new_name = EXTENSION_MAP[old_name]
+        new_name
+      else
+        old_name
+      end
+    end.compact
+  end
+  
+  # Returns two hashes, the extensions and renderer options
+  # given the extension list
+  def parse_extensions_and_renderer_options(exts)
+    exts = rename_extensions(exts)
+    exts.partition {|ext| !RENDERER_OPTIONS.include?(ext) }.
+      map {|list| list_to_truthy_hash(list) }
+  end
+  
+  # Turns a list of symbols into a hash of <tt>symbol => true</tt>.
+  def list_to_truthy_hash(list)
+    list.inject({}) {|h, k| h[k] = true; h }
+  end
 end
 

data/lib/redcarpet/render_strip.rb

@@ -0,0 +1,33 @@
+
+module Redcarpet
+  module Render
+    # Markdown-stripping renderer. Turns Markdown into plaintext
+    # Thanks to @toupeira (Markus Koller)
+    class StripDown < Base
+      # Methods where the first argument is the text content
+      [
+        # block-level calls
+        :block_code, :block_quote,
+        :block_html, :header, :list,
+        :list_item, :paragraph,
+
+        # span-level calls
+        :autolink, :codespan, :double_emphasis,
+        :emphasis, :raw_html, :triple_emphasis,
+        :strikethrough, :superscript,
+
+        # low level rendering
+        :entity, :normal_text
+      ].each do |method|
+        define_method method do |*args|
+          args.first
+        end
+      end
+
+      # Other methods where the text content is in another argument
+      def link(link, title, content)
+        content
+      end
+    end
+  end
+end

data/redcarpet.gemspec

@@ -1,7 +1,7 @@
 # encoding: utf-8
 Gem::Specification.new do |s|
   s.name = 'redcarpet'
-  s.version = '2.0.1'
+  s.version = '2.1.0'
   s.summary = "Markdown that smells nice"
   s.description = 'A fast, safe and extensible Markdown to (X)HTML parser'
   s.date = '2011-09-14'
@@ -36,6 +36,7 @@ Gem::Specification.new do |s|
     lib/redcarpet.rb
     lib/redcarpet/compat.rb
     lib/redcarpet/render_man.rb
+    lib/redcarpet/render_strip.rb
     redcarpet.gemspec
     sundown
     test/redcarpet_test.rb

data/test/redcarpet_test.rb

@@ -209,6 +209,18 @@ class MarkdownTest < Test::Unit::TestCase
       output = @markdown.render(input)
       assert_equal input.encoding.name, output.encoding.name
     end
+
+    def test_should_return_string_in_same_encoding_not_in_utf8
+      input = "testing".encode('US-ASCII')
+      output = @markdown.render(input)
+      assert_equal input.encoding.name, output.encoding.name
+    end
+    
+    def test_should_accept_non_utf8_or_ascii
+      input = "testing \xAB\xCD".force_encoding('ASCII-8BIT')
+      output = @markdown.render(input)
+      assert_equal 'ASCII-8BIT', output.encoding.name
+    end
   end
 
   def test_that_tags_can_have_dashes_and_underscores
@@ -318,6 +330,42 @@ class CustomRenderTest < Test::Unit::TestCase
   end
 end
 
+class RedcarpetCompatTest < Test::Unit::TestCase
+  def test_simple_compat_api
+    html = RedcarpetCompat.new("This is_just_a test").to_html
+    html_equal "<p>This is<em>just</em>a test</p>", html
+  end
+  
+  def test_compat_api_enables_extensions
+    html = RedcarpetCompat.new("This is_just_a test", :no_intra_emphasis).to_html
+    html_equal "<p>This is_just_a test</p>", html
+  end
+
+  def test_compat_api_knows_fenced_code_extension
+    text = "```ruby\nx = 'foo'\n```"
+    html = RedcarpetCompat.new(text, :fenced_code).to_html
+    html_equal "<pre><code class=\"ruby\">x = 'foo'\n</code></pre>", html
+  end
+  
+  def test_compat_api_ignores_gh_blockcode_extension
+    text = "```ruby\nx = 'foo'\n```"
+    html = RedcarpetCompat.new(text, :fenced_code, :gh_blockcode).to_html
+    html_equal "<pre><code class=\"ruby\">x = 'foo'\n</code></pre>", html
+  end
+
+  def test_compat_api_knows_no_intraemphasis_extension
+    html = RedcarpetCompat.new("This is_just_a test", :no_intraemphasis).to_html
+    html_equal "<p>This is_just_a test</p>", html
+  end
+  
+  def test_translate_outdated_extensions
+    # these extensions are no longer used
+    exts = [:gh_blockcode, :no_tables, :smart, :strict]
+    html = RedcarpetCompat.new('"TEST"', *exts).to_html
+    html_equal "<p>&quot;TEST&quot;</p>", html
+  end
+end
+
 # Disabled by default
 # (these are the easy ones -- the evil ones are not disclosed)
 class PathologicalInputsTest # < Test::Unit::TestCase

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: redcarpet
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 11
   prerelease: 
   segments: 
   - 2
-  - 0
   - 1
-  version: 2.0.1
+  - 0
+  version: 2.1.0
 platform: ruby
 authors: 
 - "Natacha Port\xC3\xA9"
@@ -67,6 +67,7 @@ files:
 - lib/redcarpet.rb
 - lib/redcarpet/compat.rb
 - lib/redcarpet/render_man.rb
+- lib/redcarpet/render_strip.rb
 - redcarpet.gemspec
 - test/redcarpet_test.rb
 homepage: http://github.com/tanoku/redcarpet
@@ -98,7 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Markdown that smells nice