codnar 0.1.64

data/lib/codnar/formatter.rb

@@ -0,0 +1,180 @@
+module Codnar
+
+  # Format chunks into HTML.
+  class Formatter
+
+    # Construct a Formatter based on a mapping from a classified line kind, to
+    # a Ruby expression, that converts an array of classified lines of that
+    # kind, into an array of lines of another kind. This expression is simply
+    # +eval+-ed, and is expected to make use of a variable called +lines+ that
+    # contains an array of classified lines, as produced by a Scanner. The
+    # result of evaluating the expressions is expected to be an array of any
+    # number of classified lines of any kind.
+    #
+    # Formatting repeatedly applies these formatting expressions, until the
+    # result is an array containing a single classified line, which has the
+    # kind +html+ and whose payload field contains the unified final HTML
+    # presentation of the original classified lines. In each processing round,
+    # all consecutive lines of the same kind are formated together. This allows
+    # for properly formating line kinds that use a multi-line notation such as
+    # Markdown.
+    #
+    # The default formatting expression for the kind +html+ simply joins all
+    # the payloads of all the classified lines into a single html, and returns
+    # a single "line" containing this joined HTML. All other line kinds need to
+    # have a formatting expression explicitly specified in the formatters
+    # mapping.
+    #
+    # If no formatting expression is specified for some classified line kind,
+    # an error is reported and the classified lines are wrapped in a pre HTML
+    # element with a +missing_formatter+ CSS class. Similarly, if a formatting
+    # expression fails (raises an exception), an error is reported and the
+    # lines are wrapped in a pre HTML element with a +failed_formatter+ CSS
+    # class.
+    def initialize(errors, formatters)
+      @errors = errors
+      @formatters = { "html" => "Formatter.merge_html_lines(lines)" }.merge(formatters)
+    end
+
+    # Repeatedly process an array of classified lines of arbitrary kinds until
+    # we obtain a single classified "line" containing a unified final HTML
+    # presentation of the original classified lines.
+    def lines_to_html(lines)
+      until Formatter.single_html_line?(lines)
+        lines = Grouper.lines_to_groups(lines).map { |group| process_lines_group(group) }.flatten
+      end
+      return lines.last.andand.payload.to_s
+    end
+
+  protected
+
+    # Check whether we have finally got a single HTML classified "line" for the
+    # whole classified lines sequence.
+    def self.single_html_line?(lines)
+      return lines.size <= 1 && lines[0].andand.kind == "html"
+    end
+
+    # Perform one pass of processing toward HTML on a group of consecutive
+    # classified lines with the same kind.
+    def process_lines_group(lines)
+      kind = lines.last.kind
+      formatter = @formatters[kind] ||= missing_formatter(kind)
+      begin
+        return eval formatter
+      rescue
+        return failed_formatter(lines, formatter, $!)
+      end
+    end
+
+    # Return an expression for formatting classified lines of some kind that
+    # doesn't have such a formatting expression already specified.
+    def missing_formatter(kind)
+      @errors << "No formatter specified for lines of kind: #{kind}"
+      return "Formatter.lines_to_pre_html(lines, :class => 'missing formatter error')"
+    end
+
+    # Format classified lines as HTML if the original specified formatting
+    # expression failed.
+    def failed_formatter(lines, formatter, exception)
+      @errors << "Formatter: #{formatter} for lines of kind: #{lines.last.kind} failed with exception: #{exception}"
+      return Formatter.lines_to_pre_html(lines, :class => "failed formatter error")
+    end
+
+    # {{{ Basic formatters
+
+    # Merge a group of consecutive indented lines into a group with a single
+    # classified "line". The given block is passed the joined content of all
+    # the lines, and may process it to yield the merged "line" content. If an
+    # explicit indentation is given, it overrides each line's indentation. This
+    # is useful for avoiding the inclusion of the indentation in the payload.
+    def self.merge_lines(lines, kind, indentation = nil)
+      payload = yield lines.map { |line| (indentation || line.indentation || "") + (line.payload || "") }.join("\n")
+      merged_line = lines[0]
+      merged_line.merge!("kind" => kind, "payload" => payload)
+      merged_line.delete("indentation") if indentation.nil?
+      return [ merged_line ]
+    end
+
+    # Merge a group of consecutive HTML classified lines into a group with a
+    # single HTML classified "line". This is the default formatting expression
+    # for HTML lines.
+    def self.merge_html_lines(lines)
+      return Formatter.merge_lines(lines, "html") { |payload| payload }
+    end
+
+    # Format classified lines into HTML using a pre element with optional
+    # attributes. This is the default formatting expression for classified
+    # lines of unknown kinds.
+    def self.lines_to_pre_html(lines, attributes = {})
+      return Formatter.merge_lines(lines, "html") do |payload|
+        ( "<pre" + Formatter.html_attributes(attributes) + ">\n" \
+        + CGI.escapeHTML(payload) + "\n" \
+        + "</pre>" )
+      end
+    end
+
+    # Convert an attribute mapping to HTML.
+    def self.html_attributes(attributes)
+      return "" if attributes == {}
+      return " " + attributes.map { |name, value| "#{name}='#{CGI.escapeHTML(value.to_s)}'" }.join(" ")
+    end
+
+    # Format classified lines that indicate a nested chunk to HTML.
+    def self.nested_chunk_lines_to_html(lines)
+      return lines.each do |line|
+        line.kind = "html"
+        chunk_name = line.payload
+        line.payload = "<pre class='nested chunk'>\n" \
+                     + (line.indentation || "") \
+                     + "<a class='nested chunk' href='##{chunk_name.to_id}'>#{CGI.escapeHTML(chunk_name)}</a>\n" \
+                     + "</pre>"
+        line.delete("indentation")
+      end
+    end
+
+    # Indent arbitrary HTML lines to line up with the rest of the lines.
+    def self.unindented_lines_to_html(lines)
+      merged_line = lines[0]
+      html = lines.map { |line| line.payload + "\n" }.join
+      merged_line.payload = self.indent_html(merged_line.indentation, html)
+      merged_line.kind = "html"
+      return [ merged_line ]
+    end
+
+    # Indent a chunk of HTML by some spaces. This uses a table, which is
+    # arguably the wrong way to do it.
+    def self.indent_html(indentation, html)
+      return html.chomp if indentation.nil?
+      return "<table class='layout'>\n<tr>\n" \
+           + "<td class='indentation'>\n" \
+           + "<pre>#{indentation}</pre>\n" \
+           + "</td>\n" \
+           + "<td class='html'>\n" \
+           + html \
+           + "</td>\n" \
+           + "</tr>\n</table>"
+    end
+
+    # Cast a sequence of classified lines into a different kind without
+    # any processing.
+    def self.cast_lines(lines, kind)
+      lines = lines.dup
+      lines.each { |line| line.kind = kind }
+      return lines
+    end
+
+    # Convert a sequence of marked-up classified lines to (unindented) HTML
+    def self.markup_lines_to_html(lines, klass)
+      implementation = String === klass ? Kernel.const_get(klass) : klass
+      return Formatter.merge_lines(lines, "unindented_html", "") do |payload|
+        ( "<div class='#{klass.downcase} #{lines[0].kind} markup'>\n" \
+        + implementation.to_html(payload) \
+        + "</div>" )
+      end
+    end
+
+    # }}}
+
+  end
+
+end

data/lib/codnar/grouper.rb

@@ -0,0 +1,28 @@
+module Codnar
+
+  # Group classified lines according to kind.
+  module Grouper
+
+    # Convert array of classified lines to array of classified line groups with
+    # the same line kind.
+    def self.lines_to_groups(lines)
+      groups = lines.reduce([], &method(:group_next_line))
+      return groups
+    end
+
+  protected
+
+    # Add the next classified line to the classified line groups.
+    def self.group_next_line(groups, next_line)
+      last_group = groups.last
+      if last_group.andand.last.andand.kind == next_line.kind
+        last_group.push(next_line)
+      else
+        groups.push([ next_line ])
+      end
+      return groups
+    end
+
+  end
+
+end

data/lib/codnar/gvim.rb

@@ -0,0 +1,132 @@
+module Codnar
+
+  # Syntax highlight using GVim.
+  class GVim
+
+    # Convert a sequence of classified code lines to HTML using GVim syntax
+    # highlighting. The commands array allows configuring the way that GVim
+    # will format the output (see the +cached_syntax_to_html+ method for
+    # details).
+    def self.lines_to_html(lines, syntax, commands = [])
+      return Formatter.merge_lines(lines, "html") do |payload|
+        GVim.cached_syntax_to_html(payload + "\n", syntax, commands).chomp
+      end
+    end
+
+    # The cache used for speeding up recomputing the same syntax highlighting
+    # HTML.
+    @cache = Cache.new(".gvim-cache") do |data|
+      GVim.uncached_syntax_to_html(data.text, data.syntax, data.commands)
+    end
+
+    # Force recomputation of the syntax highlighting HTML, even if a cached
+    # version exists.
+    def self.force_recompute=(force_recompute)
+      @cache.force_recompute = force_recompute
+    end
+
+    # Highlight syntax of text using GVim. This uses the GVim standard CSS
+    # classes to mark keywords, identifiers, and so on. See the GVim
+    # documentation for details. The commands array allows configuring the way
+    # that GVim will format the output. For example:
+    #
+    # * The command <tt>"+:colorscheme <name>"</tt> will override the default
+    #   color scheme used.
+    # * The command <tt>"+:let html_use_css=1"</tt> will just annotate each
+    #   HTML tag with a CSS class, instead of embedding some specific style
+    #   directly into the tag. In this case the colorscheme and background are
+    #   ignored; you will need to provide your own CSS stylesheet as part of
+    #   the final woven document to style the marked-up words.
+    #
+    # Additional commands may be useful; GVim provides a full scripting
+    # environment so there is no theoretical limit to what can be done here.
+    #
+    # Since GVim is as slow as molasses to start up, we cache the results of
+    # highlighting the syntax of each code fragment in a directory called
+    # <tt>.gvim-cache</tt>, which can appear at the current working directory
+    # or in any of its parents.
+    def self.cached_syntax_to_html(text, syntax, commands = [])
+      data = { "text" => text, "syntax" => syntax, "commands" => commands }
+      return @cache[data]
+    end
+
+    # Highlight syntax of text using GVim, without caching. This is *slow*
+    # (measured in seconds), due to GVim's start-up tim. See the
+    # +cached_syntax_to_html+ method for a faster variant and functionality
+    # details.
+    def self.uncached_syntax_to_html(text, syntax, commands = [])
+      file = write_temporary_file(text)
+      run_gvim(file, syntax, commands)
+      html = read_html_file(file)
+      delete_temporary_files(file)
+      return clean_html(html, syntax)
+    end
+
+  protected
+
+    # Write the text to highlight the syntax of into a temporary file.
+    def self.write_temporary_file(text)
+      file = Tempfile.open("codnar-")
+      file.write(text)
+      file.close(false)
+      return file
+    end
+
+    # Run GVim to highlight the syntax of a temporary file. This uses the
+    # little-known ability of GVim to emit the syntax highlighting as HTML
+    # using only command-line arguments.
+    def self.run_gvim(file, syntax, commands)
+      path = file.path
+      ENV["DISPLAY"] = "none" # Otherwise the X11 server *does* affect the result.
+      command = [
+        "gvim",
+        "-f", "-X",
+        "-u", "none",
+        "-U", "none",
+        "+:let html_ignore_folding=1",
+        "+:let use_xhtml=1",
+        "+:let html_use_css=0",
+        "+:syn on",
+        "+:set syntax=#{syntax}",
+        commands,
+        "+run! syntax/2html.vim",
+        "+:f #{path}",
+        "+:wq", "+:q",
+        path
+      ]
+      system("echo '\n' | '#{command.flatten.join("' '")}' > /dev/null 2>&1")
+    end
+
+    # Read the HTML with the syntax highlighting written out by GVim.
+    def self.read_html_file(file)
+      return File.read(html_file_path(file))
+    end
+
+    # Delete both the text and HTML temporary files.
+    def self.delete_temporary_files(file)
+      File.delete(html_file_path(file))
+      file.delete
+    end
+
+    # Find the path of the generate HTML file. You'd think it would be
+    # predictable, but it ends up either ".html" or ".xhtml" depending on the
+    # system.
+    def self.html_file_path(file)
+      return Dir.glob(file.path + ".*html")[0]
+    end
+
+    # Extract the clean highlighted syntax HTML from GVim's HTML output.
+    def self.clean_html(html, syntax)
+      if html =~ /<pre>/
+        html.sub!(/.*?<pre>/m, "<pre class='#{syntax} code syntax'>")
+        html.sub!("</body>\n</html>\n", "")
+      else
+        html.sub!(/.*?<body/m, "<div class='#{syntax} code syntax'")
+        html.sub!("</body>\n</html>\n", "</div>\n")
+      end
+      return html
+    end
+
+  end
+
+end

data/lib/codnar/hash_extensions.rb

@@ -0,0 +1,41 @@
+# Extend the core Hash class.
+class Hash
+
+  # Obtain a deep clone which shares nothing with this hash.
+  def deep_clone
+    return YAML.load(to_yaml)
+  end
+
+  # {{{ Deep merge
+
+  # Perform a deep merge with another hash.
+  def deep_merge(hash)
+    return merge(hash, &Hash::method("deep_merger"))
+  end
+
+protected
+
+  # Return a Hash merger that recursively merges nested hashes.
+  def self.deep_merger(key, default, override)
+    if Hash === default && Hash === override
+      default.deep_merge(override)
+    elsif Array === default && Array === override
+      Hash.deep_merge_arrays(default, override)
+    else
+      override
+    end
+  end
+
+  # If the overriding data array contains an empty array element ("[]"), it is
+  # replaced by the default data array being overriden.
+  def self.deep_merge_arrays(default, override)
+    embed_index = override.find_index([])
+    return override unless embed_index
+    override = override.dup
+    override[embed_index..embed_index] = default
+    return override
+  end
+
+  # }}}
+
+end

data/lib/codnar/markdown.rb

@@ -0,0 +1,47 @@
+# Extend the Markdown class.
+class Markdown
+
+  # Process a Markdown String and return the resulting HTML. In addition to the
+  # normal Markdown syntax, processing supports the following Codnar-specific
+  # extensions:
+  #
+  # * The notation "[[chunk|template]]" is expanded to embedding the specified
+  #   chunk (name) using the specified template at Weave time.
+  # * The notation "[[#name]]" defines an empty anchor. The HTML anchor id is
+  #   not the specified name, but rather the identifier generated from it (in
+  #   the same way that chunk names are converted to identifiers).
+  # * The notation "[...](#name)" defines a link to an anchor, which is either
+  #   the chunk with the specified name, or an empty anchor defined as above.
+  def self.to_html(markdown)
+    markdown = Markdown.embed_chunks(markdown)
+    markdown = Markdown.id_anchors(markdown)
+    html = RDiscount.new(markdown).to_html
+    html = Markdown.id_links(html)
+    return html.clean_markup_html
+  end
+
+protected
+
+  # Expand "[[chunk|template]]" to HTML embed tags. Use identifiers instead of
+  # names in the +src+ field for safety, unless the template is a magical file
+  # template, in which case we must preserve the file path,
+  def self.embed_chunks(markdown)
+    return markdown.gsub(/\[\[(.*?)\|(.*?)\]\]/) do
+      src = $1
+      template = $2
+      src = src.to_id unless Codnar::Weaver::FILE_TEMPLATE_PROCESSORS.include?(template)
+      "<embed src='#{src}' type='x-codnar/#{template}'/>"
+    end
+  end
+
+  # Expand "[[#name]]" anchors to HTML anchor tags with the matching identifier.
+  def self.id_anchors(markdown)
+    return markdown.gsub(/\[\[#(.*?)\]\]/) { "<a id='#{$1.to_id}'/>" }
+  end
+
+  # Expand "href='#name'" links to the matching "href='#id'" links.
+  def self.id_links(html)
+    return html.gsub(/href=(["'])#(.*?)(["'])/) { "href=#{$1}##{$2.to_id}#{$3}" }
+  end
+
+end

data/lib/codnar/merger.rb

@@ -0,0 +1,138 @@
+module Codnar
+
+  # Merge classified lines into chunks.
+  class Merger
+
+    # Convert classified lines from a disk file into chunks.
+    def self.chunks(errors, path, lines)
+      return Merger.new(errors, path, lines).chunks
+    end
+
+    # Return merged chunks containing the classified lines. Each chunk lines
+    # are only indented relative to the chunk. This allows nested chunks to be
+    # presented unindented in the final weaved HTML.
+    def chunks
+      @chunks = [ file_chunk ]
+      @stack = @chunks.dup
+      @errors.in_path(@path) { merge_lines }
+      @chunks.each { |chunk| Merger.unindent_lines(chunk.lines) }
+      return @chunks
+    end
+
+  protected
+
+    # Convert classified lines from a disk file into chunks.
+    def initialize(errors, path, lines)
+      @errors = errors
+      @path = path
+      @lines = lines
+    end
+
+    # The top-level all-the-disk-file chunk (without any classified lines)
+    def file_chunk
+      return { 
+        "name" => @path,
+        "locations" => [ { "file" => @path, "line" => 1 } ],
+        "containers" => [],
+        "contained" => [],
+        "lines" => []
+      }
+    end
+
+    # {{{ Merging nested chunk lines
+
+    # Merge all the classified lines into chunks
+    def merge_lines
+      @lines.each do |line|
+        @errors.at_line(line.number)
+        merge_line(line)
+      end
+      end_unterminated_chunks
+    end
+
+    # End all chunks missing a terminating end chunk classified line.
+    def end_unterminated_chunks
+      @stack.shift
+      @stack.each do |chunk|
+        @errors << "Missing end line for chunk: #{chunk.name}"
+      end
+    end
+
+    # Merge the next classified line.
+    def merge_line(line)
+      case line.kind
+      when "begin_chunk"
+        begin_chunk_line(line)
+      when "end_chunk"
+        end_chunk_line(line)
+      else
+        @stack.last.lines << line
+      end
+    end
+
+    # Merge a classified line that starts a new chunk.
+    def begin_chunk_line(line)
+      chunk = contained_chunk(container = @stack.last, line)
+      container.contained << chunk.name
+      container.lines << line.merge("kind" => "nested_chunk")
+      @chunks << chunk
+      @stack << chunk
+    end
+
+    # A chunk contained in another chunk.
+    def contained_chunk(container, line)
+      return {
+        "name" => new_chunk_name(line.payload),
+        "locations" => [ { "file" => @path, "line" => line.number } ],
+        "containers" => [ container.name ],
+        "contained" => [],
+        "lines" => [ line ]
+      }
+    end
+
+    # Return the name of a new chunk.
+    def new_chunk_name(name)
+      return name unless name.nil? || name == ""
+      @errors << "Begin line for chunk with no name"
+      return "#{@path}/#{@chunks.size}"
+    end
+
+    # Merge a classified line that ends an existing chunk.
+    def end_chunk_line(line)
+      return missing_begin_chunk_line(line) if @stack.size == 1
+      chunk = @stack.last
+      @errors << "End line for chunk: #{line.payload} mismatches begin line for chunk: #{chunk.name}" \
+        unless Merger.matching_end_chunk_line?(chunk, line)
+      chunk.lines << line
+      @stack.pop
+    end
+
+    # Check whether an end chunk classified line matches the begin chunk
+    # classified line.
+    def self.matching_end_chunk_line?(chunk, line)
+      line_name = line.payload
+      return line_name.to_s == "" || line_name.to_id == chunk.name.to_id
+    end
+
+    # }}}
+
+    # {{{ Unindenting chunk lines
+
+    # Remove the common indentation from a sequence of classified lines.
+    def self.unindent_lines(lines)
+      indentation = Merger.minimal_indentation(lines)
+      lines.each do |line|
+        line.indentation = line.indentation.andand.unindent(indentation)
+      end
+    end
+
+    # Find out the minimal indentation of all the classified lines.
+    def self.minimal_indentation(lines)
+      return lines.map { |line| line.indentation }.compact.min
+    end
+
+    # }}}
+
+  end
+
+end