codnar 0.1.64

data/lib/codnar/rake.rb

@@ -0,0 +1,41 @@
+require "rake"
+require "rake/tasklib"
+
+require "codnar"
+require "codnar/rake/split_task"
+require "codnar/rake/weave_task"
+
+module Codnar
+
+  # This module contains all the Codnar Rake tasks code.
+  module Rake
+
+    class << self
+
+      # The root folder to store all chunk files under.
+      attr_accessor :chunks_dir
+
+      # The list of split chunk files for later weaving.
+      attr_accessor :chunk_files
+
+    end
+
+    Rake.chunk_files = []
+    Rake.chunks_dir = "chunks"
+
+    # Compute options for invoking an application.
+    def self.application_options(output, configurations)
+      options = [ "-o", output ]
+      options += configurations.map { |configuration| [ "-c", configuration.to_s ] }.flatten
+      return options
+    end
+
+    # Return the list of actual configuration files (as opposed to names of
+    # built-in configurations) for use as dependencies.
+    def self.configuration_files(configurations)
+      return configurations.find_all { |configuration| File.exists?(configuration.to_s) }
+    end
+
+  end
+
+end

data/lib/codnar/rake/split_task.rb

@@ -0,0 +1,71 @@
+module Codnar
+
+  module Rake
+
+    # A Rake task for splitting source files to chunks.
+    class SplitTask < ::Rake::TaskLib
+
+      # Create a new Rake task for splitting source files to chunks. Each of
+      # the specified disk files is split using the specified set of
+      # configurations.
+      def initialize(paths, configurations)
+        @configurations = configurations
+        paths.each do |path|
+          define_tasks(path)
+        end
+      end
+
+    protected
+
+      # Define the tasks for splitting a single source file to chunks.
+      def define_tasks(path)
+        output = Rake.chunks_dir + "/" + path
+        define_split_file_task(path, output)
+        SplitTask.define_common_tasks
+        SplitTask.connect_common_tasks(output)
+      end
+
+      # Define the actual task for splitting the source file.
+      def define_split_file_task(path, output)
+        ::Rake::FileTask.define_task(output => [ path ] + Rake.configuration_files(@configurations)) do
+          run_split_application(path, output)
+        end
+      end
+
+      # Run the Split application for a single source file.
+      def run_split_application(path, output)
+        options = Rake.application_options(output, @configurations)
+        options << path
+        status = Application.with_argv(options) { Split.new.run }
+        raise "Codnar split errors" unless status == 0
+      end
+
+      # Define common Rake split tasks. This method may be invoked several
+      # times, only the first invocation actually defined the tasks. The common
+      # tasks are codnar_split (for splitting all the source files) and
+      # clean_codnar (for getting rid of the chunks directory).
+      def self.define_common_tasks
+        @defined_common_tasks ||= SplitTask.create_common_tasks
+      end
+
+      # Actually create common Rake split tasks.
+      def self.create_common_tasks
+        desc "Split all files into chunks"
+        ::Rake::Task.define_task("codnar_split")
+        desc "Clean all split chunks"
+        ::Rake::Task.define_task("clean_codnar") { rm_rf(Rake.chunks_dir) }
+        ::Rake::Task.define_task(:clean => "clean_codnar")
+      end
+
+      # Connect the task for splitting a single source file to the common task
+      # of splitting all source files.
+      def self.connect_common_tasks(output)
+        ::Rake::Task.define_task("codnar_split" => output)
+        Rake::chunk_files << output
+      end
+
+    end
+
+  end
+
+end

data/lib/codnar/rake/weave_task.rb

@@ -0,0 +1,59 @@
+module Codnar
+
+  module Rake
+
+    # A Rake task for weaving chunks to a single HTML.
+    class WeaveTask < ::Rake::TaskLib
+
+      # Create a Rake task for weaving chunks to a single HTML. The root source
+      # file is expected to embed all the chunks into the output HTML. The
+      # chunks are loaded from the results of all the previous created
+      # SplitTask-s.
+      def initialize(root, configurations, output = "codnar.html")
+        @root = Rake.chunks_dir + "/" + root
+        @output = output
+        @configurations = configurations
+        define_tasks
+      end
+
+    protected
+
+      # Define the tasks for weaving the chunks to a single HTML.
+      def define_tasks
+        define_weave_task
+        connect_common_tasks
+      end
+
+      # Define the actual task for weaving the chunks to a single HTML.
+      def define_weave_task
+        desc "Weave chunks into HTML" unless ::Rake.application.last_comment
+        ::Rake::Task.define_task("codnar_weave" => @output)
+        ::Rake::FileTask.define_task(@output => Rake.chunk_files + Rake.configuration_files(@configurations)) do
+          run_weave_application
+        end
+      end
+
+      # Run the Weave application for a single source file.
+      def run_weave_application
+        options = Rake.application_options(@output, @configurations)
+        options << @root
+        options += Rake.chunk_files.reject { |chunk| chunk == @root }
+        status = Application.with_argv(options) { Weave.new.run }
+        raise "Codnar weave errors" unless status == 0
+      end
+
+      # Connect the task for cleaning up after weaving (+clobber_codnar+) to the
+      # common task of cleaning up everything (+clobber+).
+      def connect_common_tasks
+        desc "Build the code narrative HTML"
+        ::Rake::Task.define_task(:codnar => "codnar_weave")
+        desc "Remove woven HTML documentation"
+        ::Rake::Task.define_task("clobber_codnar") { rm_rf(@output) }
+        ::Rake::Task.define_task(:clobber => "clobber_codnar")
+      end
+
+    end
+
+  end
+
+end

data/lib/codnar/rdoc.rb

@@ -0,0 +1,9 @@
+# Extend the RDoc module.
+module RDoc
+
+  # Process a RDoc String and return the resulting HTML.
+  def self.to_html(rdoc)
+    return ::RDoc::Markup::ToHtml.new.convert(rdoc).clean_markup_html
+  end
+
+end

data/lib/codnar/reader.rb

@@ -0,0 +1,121 @@
+module Codnar
+
+  # Read chunks from disk files.
+  class Reader
+
+    # Load all chunks from the specified disk files to memory for later access
+    # by name.
+    def initialize(errors, paths)
+      @errors = errors
+      @chunks = {}
+      @used = {}
+      paths.each do |path|
+        read_path_chunks(path)
+      end
+    end
+
+    # Fetch a chunk by its name.
+    def [](name)
+      id = name.to_id
+      @used[id] = true
+      return @chunks[id] ||= (
+        @errors << "Missing chunk: #{name}"
+        Reader.fake_chunk(name)
+      )
+    end
+
+    # Collect errors for unused chunks.
+    def collect_unused_chunk_errors
+      @chunks.each do |id, chunk|
+        @errors.push("#{$0}: Unused chunk: #{chunk.name} #{Reader.locations_message(chunk)}") unless @used[id]
+      end
+    end
+
+  protected
+
+    # Load and merge all chunks from a disk file into memory.
+    def read_path_chunks(path)
+      @errors.in_path(path) do
+        chunks = load_path_chunks(path)
+        next unless chunks
+        merge_loaded_chunks(chunks)
+        @root_chunk ||= chunks[0].name
+      end
+    end
+
+    # Load all chunks from a disk file into memory.
+    def load_path_chunks(path)
+      chunks = YAML.load_file(path)
+      @errors << "Invalid chunks data" unless chunks
+      # TODO: A bit more validation would be nice.
+      return chunks
+    end
+
+    # Merge an array of chunks into memory.
+    def merge_loaded_chunks(chunks)
+      chunks.each do |new_chunk|
+        old_chunk = @chunks[id = new_chunk.name.to_id]
+        if old_chunk.nil?
+          @chunks[id] = new_chunk
+        elsif Reader.same_chunk?(old_chunk, new_chunk)
+          Reader.merge_same_chunks(old_chunk, new_chunk)
+        else
+          @errors.push(Reader.different_chunks_error(old_chunk, new_chunk))
+        end
+      end
+    end
+
+    # Merge a new "same" chunk into an old one.
+    def self.merge_same_chunks(old_chunk, new_chunk)
+      old_chunk.locations = \
+        (old_chunk.locations + new_chunk.locations).uniq.sort \
+          do |first_location, second_location|
+            [ first_location.file.to_id, first_location.line ] \
+            <=> [ second_location.file.to_id, second_location.line ]
+          end
+      old_chunk.containers = \
+        (old_chunk.containers + new_chunk.containers).uniq.sort \
+          do |first_name, second_name|
+            first_name.to_id <=> second_name.to_id
+          end
+    end
+
+    # Check whether two chunks contain the same "stuff".
+    def self.same_chunk?(old_chunk, new_chunk)
+      return Reader.chunk_payload(old_chunk) == Reader.chunk_payload(new_chunk)
+    end
+
+    # Return just the actual payload of a chunk for equality comparison.
+    def self.chunk_payload(chunk)
+      chunk = chunk.reject { |key, value| [ "locations", "name", "containers" ].include?(key) }
+      chunk.contained.map! { |name| name.to_id }
+      return chunk
+    end
+
+    # Error message when two different chunks have the same name.
+    def self.different_chunks_error(old_chunk, new_chunk)
+      old_location = Reader.locations_message(old_chunk)
+      new_location = Reader.locations_message(new_chunk)
+      return "#{$0}: Chunk: #{old_chunk.name} is different #{new_location}, and #{old_location}"
+    end
+
+    # Format a chunk's location for an error message.
+    def self.locations_message(chunk)
+      locations = chunk.locations.map { |location| "in file: #{location.file} at line: #{location.line}" }
+      return locations.join(" or ")
+    end
+
+    # Return a fake chunk for the specified name.
+    def self.fake_chunk(name)
+      return {
+        "name" => name,
+        "locations" => [ { "file" => "MISSING" } ],
+        "contained" => [],
+        "containers" => [],
+        "html" => "<div class='missing chunk error'>\nMISSING\n</div>"
+      }
+    end
+
+  end
+
+end

data/lib/codnar/scanner.rb

@@ -0,0 +1,216 @@
+module Codnar
+
+  # Scan a file into classified lines.
+  class Scanner
+
+    # Construct a scanner based on a syntax in the following structure:
+    #
+    #   patterns:
+    #     <name>:
+    #       name: <name>
+    #       kind: <kind>
+    #       regexp: <regexp>
+    #       groups:
+    #       - <name>
+    #   states:
+    #     <name>:
+    #       name: <name>
+    #       transitions:
+    #       - pattern: <pattern>
+    #         kind: <kind>
+    #         next_state: <state>
+    #   start_state: <state>
+    #
+    # To allow for cleaner YAML files to specify the syntax, the following
+    # shorthands are supported:
+    #
+    # - A pattern or state reference can be presented by the string name of the
+    #   pattern or state.
+    # - The name field of a state or pattern can be ommitted. If specified, it
+    #   must be identical to the key in the states or patterns mapping.
+    # - The kind field of a pattern can be ommitted; by default it is assumed
+    #   to be identical to the pattern name.
+    # - A pattern regexp can be presented by a plain string.
+    # - The pattern groups field can be ommitted or contain +nil+ if it is
+    #   equal to [ "indentation", "payload" ].
+    # - The kind field of a transition can be ommitted; by default it is
+    #   assumed to be identical to the pattern kind.
+    # - The next state of a transition can be ommitted; by default it is
+    #   assumed to be identical to the containing state.
+    # - The start state can be ommitted; by default it is assumed to be named
+    #   +start+.
+    #
+    # When the Scanner is constructed, a deep clone of the syntax object is
+    # created and modified to expand all the above shorthands. Any problems
+    # detected during this process are pushed into the errors.
+    def initialize(errors, syntax)
+      @errors = errors
+      @syntax = syntax.deep_clone
+      @syntax.patterns.each { |name, pattern| expand_pattern_shorthands(name, pattern) }
+      @syntax.states.each { |name, state| expand_state_shorthands(name, state) }
+      @syntax.start_state = resolve_start_state
+    end
+
+    # Scan a disk file into classified lines in the following format (where the
+    # groups contain the text extracted by the matching pattern):
+    #
+    #   - kind: <kind>
+    #     line: <text>
+    #     <group>: <text>
+    #
+    # By convention, each classified line has a "payload" group that contains
+    # the "main" content of the line (chunk name for begin/end/nested chunk
+    # lines, clean comment text for comment lines, etc.). In addition, most
+    # classified lines have an "indentation" group that contains the leading
+    # white space (which is not included in the payload).
+    #
+    # If at some state, a file line does not match any pattern, the scanner
+    # will push a message into the errors. In addition it will classify the
+    # line as follows:
+    #
+    #   - kind: error
+    #     state: <name>
+    #     line: <text>
+    #     indentation: <leading white space>
+    #     payload: <line text following the indentation>
+    def lines(path)
+      @path = path
+      @lines = []
+      @state = @syntax.start_state
+      @errors.in_path(path) { scan_path }
+      return @lines
+    end
+
+  protected
+
+    # {{{ Scanner pattern shorthands
+
+    # Expand all the shorthands used in the pattern.
+    def expand_pattern_shorthands(name, pattern)
+      pattern.kind ||= fill_name(name, pattern, "Pattern")
+      pattern.groups ||= [ "indentation", "payload" ]
+      pattern.regexp = convert_to_regexp(name, pattern.regexp)
+    end
+
+    # Convert a string regexp to a real Regexp.
+    def convert_to_regexp(name, regexp)
+      return regexp if Regexp == regexp
+      begin
+        return Regexp.new(regexp)
+      rescue
+        @errors << "Invalid pattern: #{name} regexp: #{regexp} error: #{$!}"
+      end
+    end
+
+    # Fill in the name field for state or pattern object.
+    def fill_name(name, data, type)
+      data_name = data.name ||= name
+      @errors << "#{type}: #{name} has wrong name: #{data_name}" if data_name != name
+      return data_name
+    end
+
+    # }}}
+
+    # {{{ Scanner state shorthands
+
+    # Expand all the shorthands used in the state.
+    def expand_state_shorthands(name, state)
+      fill_name(name, state, "State")
+      state.transitions.each do |transition|
+        pattern = transition.pattern = lookup(@syntax.patterns, "pattern", transition.pattern)
+        transition.kind ||= pattern.andand.kind
+        transition.next_state = lookup(@syntax.states, "state", transition.next_state || state)
+      end
+    end
+
+    # Convert a string name to an actual data reference.
+    def lookup(mapping, type, reference)
+      return reference unless String === reference
+      data = mapping[reference]
+      @errors << "Reference to a missing #{type}: #{reference}" unless data
+      return data
+    end
+
+    # Resolve the start state reference.
+    def resolve_start_state
+      return lookup(@syntax.states, "state", @syntax.start_state || "start") || {
+        "name" => "missing_start_state",
+        "kind" => "error",
+        "transitions" => []
+      }
+    end
+
+    # }}}
+
+    # {{{ Scanner file processing
+
+    # Scan a disk file.
+    def scan_path
+      File.open(@path, "r") do |file|
+        scan_file(file)
+      end
+    end
+
+    # Scan an opened file.
+    def scan_file(file)
+      @line_number = 0
+      file.read.each_line do |line|
+        @errors.at_line(@line_number += 1)
+        scan_line(line.chomp)
+      end
+    end
+
+    # Scan the next file line.
+    def scan_line(line)
+      @state.transitions.each do |transition|
+        return if transition.pattern && transition.next_state && classify_matching_line(line, transition)
+      end
+      unclassified_line(line, @state.name)
+    end
+
+    # }}}
+
+    # {{{ Scanner line processing
+
+    # Handle a file line, only if it matches the pattern.
+    def classify_matching_line(line, transition)
+      match = (pattern = transition.pattern).regexp.match(line)
+      return false unless match
+      @lines << Scanner.extracted_groups(match, pattern.groups).update({
+        "line" => line,
+        "kind" => transition.kind,
+        "number" => @line_number
+      })
+      @state = transition.next_state
+      return true
+    end
+
+    # Extract named groups from a match. As a special case, indentation is
+    # deleted if there is no payload.
+    def self.extracted_groups(match, groups)
+      extracted = {}
+      groups.each_with_index do |group, index|
+        extracted[group] = match[index + 1]
+      end
+      extracted.delete("indentation") if match[0] == ""
+      return extracted
+    end
+
+    # Handle a file line that couldn't be classified.
+    def unclassified_line(line, state_name)
+      @lines << {
+        "line" => line,
+        "indentation" => line.indentation,
+        "payload" => line.unindent,
+        "kind" => "error",
+        "state" => state_name,
+        "number" => @line_number
+      }
+      @errors << "State: #{state_name} failed to classify line: #{@lines.last.payload}"
+    end
+
+    # }}}
+
+  end
+
+end