hamdown_core 0.5.0

data/lib/hamdown_core/indent_tracker.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+require_relative 'error'
+
+module HamdownCore
+  class IndentTracker
+    class IndentMismatch < Error
+      attr_reader :current_level, :indent_levels
+
+      def initialize(current_level, indent_levels, lineno)
+        super("Unexpected indent level: #{current_level}: indent_level=#{indent_levels}", lineno)
+        @current_level = current_level
+        @indent_levels = indent_levels
+      end
+    end
+
+    class InconsistentIndent < Error
+      attr_reader :previous_size, :current_size
+
+      def initialize(previous_size, current_size, lineno)
+        super("Inconsistent indentation: #{current_size} spaces used for indentation, but the rest of the document was indented using #{previous_size} spaces.", lineno)
+        @previous_size = previous_size
+        @current_size = current_size
+      end
+    end
+
+    class HardTabNotAllowed < Error
+      def initialize(lineno)
+        super('Indentation with hard tabs are not allowed :-p', lineno)
+      end
+    end
+
+    def initialize(on_enter: nil, on_leave: nil)
+      @indent_levels = [0]
+      @on_enter = on_enter || lambda { |_level, _text| }
+      @on_leave = on_leave || lambda { |_level, _text| }
+      @comment_level = nil
+    end
+
+    def process(line, lineno)
+      if line.include?("\t")
+        raise HardTabNotAllowed.new(lineno)
+      end
+      indent, text = split(line)
+      indent_level = indent.size
+
+      unless text.empty?
+        track(indent_level, text, lineno)
+      end
+      [text, indent]
+    end
+
+    def split(line)
+      m = line.match(/\A( *)(.*)\z/)
+      [m[1], m[2]]
+    end
+
+    def finish
+      indent_leave(0, '', -1)
+    end
+
+    def current_level
+      @indent_levels.last
+    end
+
+    def enter_comment!
+      @comment_level = @indent_levels[-2]
+    end
+
+    def check_indent_level!(lineno)
+      if @indent_levels.size >= 3
+        previous_size = @indent_levels[-2] - @indent_levels[-3]
+        current_size = @indent_levels[-1] - @indent_levels[-2]
+        if previous_size != current_size
+          raise InconsistentIndent.new(previous_size, current_size, lineno)
+        end
+      end
+    end
+
+    private
+
+    def track(indent_level, text, lineno)
+      if indent_level > @indent_levels.last
+        indent_enter(indent_level, text, lineno)
+      elsif indent_level < @indent_levels.last
+        indent_leave(indent_level, text, lineno)
+      end
+    end
+
+    def indent_enter(indent_level, text, _lineno)
+      unless @comment_level
+        @indent_levels.push(indent_level)
+        @on_enter.call(indent_level, text)
+      end
+    end
+
+    def indent_leave(indent_level, text, lineno)
+      if @comment_level
+        if indent_level <= @comment_level
+          # finish comment mode
+          @comment_level = nil
+        else
+          # still in comment
+          return
+        end
+      end
+
+      while indent_level < @indent_levels.last
+        @indent_levels.pop
+        @on_leave.call(indent_level, text)
+      end
+
+      if indent_level != @indent_levels.last
+        raise IndentMismatch.new(indent_level, @indent_levels.dup, lineno)
+      end
+    end
+  end
+end

data/lib/hamdown_core/line_parser.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+module HamdownCore
+  class LineParser
+    attr_reader :filename, :lineno
+
+    def initialize(filename, template_str)
+      @filename = filename
+      @lines = template_str.each_line.map { |line| line.chomp.rstrip }
+      @lineno = 0
+    end
+
+    def next_line(in_filter: false)
+      line = move_next
+      if !in_filter && is_multiline?(line)
+        next_multiline(line)
+      else
+        line
+      end
+    end
+
+    def has_next?
+      @lineno < @lines.size
+    end
+
+    private
+
+    MULTILINE_SUFFIX = ' |'
+
+    # Regex to check for blocks with spaces around arguments. Not to be confused
+    # with multiline script.
+    # For example:
+    #     foo.each do | bar |
+    #       = bar
+    #
+    BLOCK_WITH_SPACES = /do\s*\|\s*[^\|]*\s+\|\z/o
+
+    def is_multiline?(line)
+      line = line.lstrip
+      line.end_with?(MULTILINE_SUFFIX) && line !~ BLOCK_WITH_SPACES
+    end
+
+    def move_next
+      @lines[@lineno].tap do
+        @lineno += 1
+      end
+    end
+
+    def move_back
+      @lineno -= 1
+    end
+
+    def next_multiline(line)
+      buf = [line[0, line.size - 1]]
+      while @lineno < @lines.size
+        line = move_next
+
+        if is_multiline?(line)
+          line = line[0, line.size - 1]
+          buf << line.lstrip
+        else
+          move_back
+          break
+        end
+      end
+      buf.join("\n")
+    end
+  end
+end

data/lib/hamdown_core/parser.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+require_relative 'ast'
+require_relative 'element_parser'
+require_relative 'error'
+require_relative 'filter_parser'
+require_relative 'indent_tracker'
+require_relative 'line_parser'
+require_relative 'ruby_multiline'
+require_relative 'script_parser'
+require_relative 'utils'
+require 'pry'
+
+module HamdownCore
+  class Parser
+    def initialize(options = {})
+      @filename = options[:filename]
+    end
+
+    def call(template_str)
+      @ast = Ast::Root.new
+      @stack = []
+      @line_parser = LineParser.new(@filename, template_str)
+      @indent_tracker = IndentTracker.new(on_enter: method(:indent_enter), on_leave: method(:indent_leave))
+      @filter_parser = FilterParser.new(@indent_tracker)
+
+      while @line_parser.has_next?
+        in_filter = !@ast.is_a?(Ast::HamlComment) && @filter_parser.enabled?
+        line = @line_parser.next_line(in_filter: in_filter)
+        if in_filter
+          ast = @filter_parser.append(line)
+          if ast
+            @ast << ast
+          end
+        end
+        unless @filter_parser.enabled?
+          line_count = line.count("\n")
+          line.delete!("\n")
+          parse_line(line)
+          line_count.times do
+            @ast << create_node(Ast::Empty)
+          end
+        end
+      end
+
+      ast = @filter_parser.finish
+      if ast
+        @ast << ast
+      end
+      @indent_tracker.finish
+      @ast
+    rescue Error => e
+      if @filename && e.lineno
+        e.backtrace.unshift "#{@filename}:#{e.lineno}"
+      end
+      raise e
+    end
+
+    private
+
+    DOCTYPE_PREFIX = '!'
+    ELEMENT_PREFIX = '%'
+    COMMENT_PREFIX = '/'
+    SILENT_SCRIPT_PREFIX = '-'
+    DIV_ID_PREFIX = '#'
+    DIV_CLASS_PREFIX = '.'
+    FILTER_PREFIX = ':'
+    ESCAPE_PREFIX = '\\'
+
+    MARKDOWN = {
+      'headers' => /^\#{1,6} .+$/,
+      'list_item' => /((^[\d{1,3}|\*|\+]\.? .*\n?)+)/, # not nesting
+      'quotes' => /((^>{1,4} .+\n?)+)/,
+      'image' => /^!\[[^\[\]]*?\]\([^\s]*?\)/,
+      'image_title' => /^!\[[^\[\]]*?\]\([^\s]*\s\".*\"\)/,
+      'link' => /^[^!]\[[^\[\]]*?\]\([^\s]*?\)/,
+      'link_with_title' => /^[^!]\[[^\[\]]*?\]\([^\s]*\s\".*\"\)/
+      # codeblock # nesting within plain text is illegal
+    }
+    # bold, italic, b_italic, monospace
+    # paragraphs !
+
+    def parse_line(line)
+      text, indent = @indent_tracker.process(line, @line_parser.lineno)
+
+      if text.empty?
+        @ast << create_node(Ast::Empty)
+        return
+      end
+
+      if @ast.is_a?(Ast::HamlComment)
+        @ast << create_node(Ast::Text) { |t| t.text = text }
+        return
+      end
+
+      case text
+      when MARKDOWN['headers']
+        parse_md_header(text)
+      when MARKDOWN['list_item']
+        parse_md_list(text)
+      when MARKDOWN['quotes']
+        parse_md_quote(text)
+      when MARKDOWN['image']
+        parse_md_image(text)
+      when MARKDOWN['image_title']
+        parse_md_image(text, true)
+      when MARKDOWN['link']
+        parse_md_link(text)
+      when MARKDOWN['link_title']
+        parse_md_link(text, true)
+      else
+        std_parse_line(text, indent)
+      end
+    end
+
+    def std_parse_line(text, indent)
+      case text[0]
+      when ESCAPE_PREFIX
+        parse_plain(text[1..-1])
+      when ELEMENT_PREFIX
+        parse_element(text)
+      when DOCTYPE_PREFIX
+        if text.start_with?('!!!')
+          parse_doctype(text)
+        else
+          parse_script(text)
+        end
+      when COMMENT_PREFIX
+        parse_comment(text)
+      when SILENT_SCRIPT_PREFIX
+        parse_silent_script(text)
+      when DIV_ID_PREFIX, DIV_CLASS_PREFIX
+        if text.start_with?('#{')
+          parse_script(text)
+        else
+          parse_line("#{indent}%div#{text}")
+        end
+      when FILTER_PREFIX
+        parse_filter(text)
+      else
+        parse_script(text)
+      end
+    end
+
+    def parse_doctype(text)
+      @ast << create_node(Ast::Doctype) { |d| d.doctype = text[3..-1].strip }
+    end
+
+    def parse_comment(text)
+      text = text[1, text.size - 1].strip
+      comment = create_node(Ast::HtmlComment)
+      comment.comment = text
+      if text[0] == '['
+        comment.conditional, rest = parse_conditional_comment(text)
+        text.replace(rest)
+      end
+      @ast << comment
+    end
+
+    CONDITIONAL_COMMENT_REGEX = /[\[\]]/o
+
+    def parse_conditional_comment(text)
+      s = StringScanner.new(text[1..-1])
+      depth = Utils.balance(s, '[', ']')
+      if depth == 0
+        [s.pre_match, s.rest.lstrip]
+      else
+        syntax_error!('Unmatched brackets in conditional comment')
+      end
+    end
+
+    def parse_plain(text)
+      @ast << create_node(Ast::Text) { |t| t.text = text }
+    end
+
+    def parse_md_header(text)
+      @ast << create_node(Ast::MdHeader) { |t| t.text = text }
+    end
+
+    def parse_md_list(text)
+      @ast << create_node(Ast::MdList) { |t| t.text = text }
+    end
+
+    def parse_md_quote(text)
+      @ast << create_node(Ast::MdQuote) { |t| t.text = text }
+    end
+
+    def parse_md_image(text, title = false)
+      if title == true
+        @ast << create_node(Ast::MdImageTitle) { |t| t.text = text }
+      else
+        @ast << create_node(Ast::MdImage) { |t| t.text = text }
+      end
+    end
+
+    def parse_md_link(text, title = false)
+      if title == true
+        @ast << create_node(Ast::MdLinkTitle) { |t| t.text = text }
+      else
+        @ast << create_node(Ast::MdLink) { |t| t.text = text }
+      end
+    end
+
+    def parse_element(text)
+      @ast << ElementParser.new(@line_parser).parse(text)
+    end
+
+    def parse_script(text)
+      node = ScriptParser.new(@line_parser).parse(text)
+      if node.is_a?(Ast::Script)
+        node.keyword = block_keyword(node.script)
+      end
+      @ast << node
+    end
+
+    def parse_silent_script(text)
+      if text.start_with?('-#')
+        @ast << create_node(Ast::HamlComment)
+        return
+      end
+      node = create_node(Ast::SilentScript)
+      script = text[/\A- *(.*)\z/, 1]
+      node.script = [script, *RubyMultiline.read(@line_parser, script)].join("\n")
+      node.keyword = block_keyword(node.script)
+      @ast << node
+    end
+
+    def parse_filter(text)
+      filter_name = text[/\A#{FILTER_PREFIX}(\w+)\z/, 1]
+      unless filter_name
+        syntax_error!("Invalid filter name: #{text}")
+      end
+      @filter_parser.start(filter_name, @line_parser.filename, @line_parser.lineno)
+    end
+
+    def indent_enter(_, _text)
+      empty_lines = []
+      while @ast.children.last.is_a?(Ast::Empty)
+        empty_lines << @ast.children.pop
+      end
+      @stack.push(@ast)
+      @ast = @ast.children.last
+      case @ast
+      when Ast::Text
+        syntax_error!('nesting within plain text is illegal')
+      when Ast::Doctype
+        syntax_error!('nesting within a header command is illegal')
+      when nil
+        syntax_error!('Indenting at the beginning of the document is illegal')
+      end
+      @ast.children = empty_lines
+      if @ast.is_a?(Ast::Element) && @ast.self_closing
+        syntax_error!('Illegal nesting: nesting within a self-closing tag is illegal')
+      end
+      if @ast.is_a?(Ast::HtmlComment) && !@ast.comment.empty?
+        syntax_error!('Illegal nesting: nesting within a html comment that already has content is illegal.')
+      end
+      if @ast.is_a?(Ast::HamlComment)
+        @indent_tracker.enter_comment!
+      else
+        @indent_tracker.check_indent_level!(@line_parser.lineno)
+      end
+      nil
+    end
+
+    def indent_leave(_indent_level, _text)
+      parent_ast = @stack.pop
+      @ast = parent_ast
+      nil
+    end
+
+    MID_BLOCK_KEYWORDS = %w[else elsif rescue ensure end when].freeze
+    START_BLOCK_KEYWORDS = %w[if begin case unless].freeze
+    # Try to parse assignments to block starters as best as possible
+    START_BLOCK_KEYWORD_REGEX = /(?:\w+(?:,\s*\w+)*\s*=\s*)?(#{Regexp.union(START_BLOCK_KEYWORDS)})/
+    BLOCK_KEYWORD_REGEX = /^-?\s*(?:(#{Regexp.union(MID_BLOCK_KEYWORDS)})|#{START_BLOCK_KEYWORD_REGEX.source})\b/
+
+    def block_keyword(text)
+      m = text.match(BLOCK_KEYWORD_REGEX)
+      if m
+        m[1] || m[2]
+      end
+    end
+
+    def syntax_error!(message)
+      raise Error.new(message, @line_parser.lineno)
+    end
+
+    def create_node(klass, &block)
+      klass.new.tap do |node|
+        node.filename = @line_parser.filename
+        node.lineno = @line_parser.lineno
+        if block
+          yield(node)
+        end
+      end
+    end
+  end
+end

data/lib/hamdown_core/ruby_multiline.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+module HamdownCore
+  module RubyMultiline
+    def self.read(line_parser, current_text)
+      buf = []
+      while is_ruby_multiline?(current_text)
+        current_text = line_parser.next_line
+        buf << current_text
+      end
+      buf
+    end
+
+    # `text' is a Ruby multiline block if it:
+    # - ends with a comma
+    # - but not "?," which is a character literal
+    #   (however, "x?," is a method call and not a literal)
+    # - and not "?\," which is a character literal
+    def self.is_ruby_multiline?(text)
+      text && text.length > 1 && text[-1] == ',' &&
+        !((text[-3, 2] =~ /\W\?/) || text[-3, 2] == '?\\')
+    end
+    private_class_method :is_ruby_multiline?
+  end
+end

data/lib/hamdown_core/script_parser.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+require_relative 'ast'
+require_relative 'error'
+require_relative 'ruby_multiline'
+
+module HamdownCore
+  class ScriptParser
+    def initialize(line_parser)
+      @line_parser = line_parser
+    end
+
+    def parse(text)
+      case text[0]
+      when '=', '~'
+        parse_script(text)
+      when '&'
+        parse_sanitized(text)
+      when '!'
+        parse_unescape(text)
+      else
+        parse_text(text)
+      end
+    end
+
+    private
+
+    def parse_script(text)
+      if text[1] == '='
+        create_node(Ast::Text) { |t| t.text = text[2..-1].strip }
+      else
+        node = create_node(Ast::Script)
+        script = text[1..-1].lstrip
+        if script.empty?
+          syntax_error!('No Ruby code to evaluate')
+        end
+        node.script = [script, *RubyMultiline.read(@line_parser, script)].join("\n")
+        node.preserve = text[0] == '~'
+        node
+      end
+    end
+
+    def parse_sanitized(text)
+      if text.start_with?('&==')
+        create_node(Ast::Text) { |t| t.text = text[3..-1].lstrip }
+      elsif text[1] == '=' || text[1] == '~'
+        node = create_node(Ast::Script)
+        script = text[2..-1].lstrip
+        if script.empty?
+          syntax_error!('No Ruby code to evaluate')
+        end
+        node.script = [script, *RubyMultiline.read(@line_parser, script)].join("\n")
+        node.preserve = text[1] == '~'
+        node
+      else
+        create_node(Ast::Text) { |t| t.text = text[1..-1].strip }
+      end
+    end
+
+    def parse_unescape(text)
+      if text.start_with?('!==')
+        create_node(Ast::Text) do |t|
+          t.text = text[3..-1].lstrip
+          t.escape_html = false
+        end
+      elsif text[1] == '=' || text[1] == '~'
+        node = create_node(Ast::Script)
+        node.escape_html = false
+        script = text[2..-1].lstrip
+        if script.empty?
+          syntax_error!('No Ruby code to evaluate')
+        end
+        node.script = [script, *RubyMultiline.read(@line_parser, script)].join("\n")
+        node.preserve = text[1] == '~'
+        node
+      else
+        create_node(Ast::Text) do |t|
+          t.text = text[1..-1].lstrip
+          t.escape_html = false
+        end
+      end
+    end
+
+    def parse_text(text)
+      text = text.lstrip
+      if text.empty?
+        nil
+      else
+        create_node(Ast::Text) { |t| t.text = text }
+      end
+    end
+
+    def syntax_error!(message)
+      raise Error.new(message, @line_parser.lineno)
+    end
+
+    def create_node(klass, &block)
+      klass.new.tap do |node|
+        node.filename = @line_parser.filename
+        node.lineno = @line_parser.lineno
+        if block
+          block.call(node)
+        end
+      end
+    end
+  end
+end

data/lib/hamdown_core/transformer.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module HamdownCore
+  module Transformer
+    class << self
+      def call(ast)
+        transform(ast)
+      end
+
+      private
+
+      def transform(root_node)
+        collected_nodes = []
+        new_root_node = root_node.clone
+        new_root_node.children = []
+        root_node.children.each do |node|
+          if node.respond_to?(:markdownable?) && node.markdownable?
+            collected_nodes << node
+            next
+          end
+
+          if collected_nodes.size > 0
+            filter = create_filter(collected_nodes)
+            collected_nodes = []
+            new_root_node << filter
+          end
+
+          if node.respond_to?(:children) && node.children.size > 0
+            node = transform(node)
+          end
+
+          new_root_node << node
+        end
+        if collected_nodes.size > 0
+          filter = create_filter(collected_nodes)
+          collected_nodes = []
+          new_root_node << filter
+        end
+        new_root_node
+      end
+
+      def create_filter(nodes)
+        filter = HamdownCore::Ast::Filter.new
+        filter.name = 'markdown'
+        nodes.each do |collected_node|
+          filter.texts << collected_node.text
+        end
+        filter
+      end
+    end
+  end
+end

data/lib/hamdown_core/utils.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+module HamdownCore
+  module Utils
+    module_function
+
+    def balance(scanner, start, finish, depth = 1)
+      re = /(#{Regexp.escape(start)}|#{Regexp.escape(finish)})/
+      while depth > 0 && scanner.scan_until(re)
+        if scanner.matched == start
+          depth += 1
+        else
+          depth -= 1
+        end
+      end
+      depth
+    end
+  end
+end

data/lib/hamdown_core/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module HamdownCore
+  VERSION = '0.5.0'
+end

data/lib/hamdown_core.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+require_relative 'hamdown_core/version'
+require_relative 'hamdown_core/cli'
+require_relative 'hamdown_core/engine'
+
+module HamdownCore
+  # Your code goes here...
+end