markdown_record 0.1.3

data/lib/markdown_record/json_renderer.rb

@@ -0,0 +1,270 @@
+require "redcarpet"
+require "redcarpet/render_strip"
+
+module MarkdownRecord
+  class JsonRenderer
+    include ::MarkdownRecord::PathUtilities
+    include ::MarkdownRecord::ContentDsl
+
+    def initialize(file_saver: ::MarkdownRecord::FileSaver.new)
+      @indexer = MarkdownRecord::Indexer.new
+      @json_models = {}
+      @described_models = []
+      @file_saver = file_saver
+    end
+
+    def render_models_for_subdirectory(subdirectory: "", **options)
+      @directory_meta = {}
+      content = @indexer.index(subdirectory: subdirectory)
+
+      rendered_subdirectory = base_content_path.join(subdirectory)
+      json_hash, _ = render_models_recursively(content, rendered_subdirectory, options)
+
+      save_content_recursively(json_hash, options, Pathname.new(""))
+      json_hash
+    end
+
+    def render_models_for_file(file_path:, **options)
+      file = @indexer.file(file_path)
+      
+      # render json models
+      json = render_models(file, file_path, options)
+      
+      # save json
+      @file_saver.save_to_file(json.to_json, "#{clean_path(file_path)}.json", options)
+      
+      json
+    end
+
+  private
+
+    def render_models_recursively(file_or_directory, full_path, options)
+      case file_or_directory.class.name;
+      when Hash.name # if it is a directory
+        directory_hash, concat_hash = *render_nested_models(file_or_directory, full_path, options)
+
+        # concatenate child hashes if :concat = true
+        if options[:concat]
+          concatenate_nested_models(full_path, directory_hash, concat_hash, options)
+        end
+
+        [directory_hash, concat_hash]
+      when String.name # if it is a file
+        content = render_models(file_or_directory, full_path, options)
+
+        if options[:deep] && options[:render_content_fragment_json]
+          add_content_fragment_for_file(content, full_path, false, @fragment_meta)
+        end
+
+        content_hash = { full_path.basename.to_s => content }
+        result = [{}, {}]
+        result[0] = content_hash if options[:deep]
+        result[1] = content_hash if options[:concat]
+        result
+      end
+    end
+
+    def render_nested_models(file_or_directory, full_path, options)
+      directory_hash = { full_path.basename.to_s => {} } # hash representing the directory and its contents
+      concat_hash = { full_path.basename.to_s => {} }
+      # iterate through directory contents
+      file_or_directory.each do |child_file_or_directory_name, child_file_or_directory|
+        
+        # get full path for next recursion
+        child_full_path = full_path.join(child_file_or_directory_name)
+        # get response from next recursion
+        child_content_hash, child_concat_hash = render_models_recursively(
+                                                  child_file_or_directory, 
+                                                  child_full_path, 
+                                                  options
+                                                )
+        # merge response hashes                                                  
+        concat_hash[full_path.basename.to_s].merge!(child_concat_hash)
+        directory_hash[full_path.basename.to_s].merge!(child_content_hash)
+      end
+
+      [directory_hash, concat_hash]
+    end
+
+    def concatenate_nested_models(full_path, directory_hash, concat_hash, options)
+      concatenated_json = {}
+      concatenate_json_recursively(concat_hash, concatenated_json)
+ 
+      # add content fragments if render_content_fragment_json = true
+      if options[:render_content_fragment_json]
+        filename, subdirectory = full_path_to_parts(full_path)
+        directory_meta = @directory_meta[clean_path("#{subdirectory}/#{filename}")] || {}
+
+        add_content_fragment_for_file(concat_hash[full_path.basename.to_s], full_path, true, directory_meta)
+        add_content_fragment_for_file(concatenated_json, full_path, true, directory_meta)
+      end
+
+      directory_hash["#{full_path.basename.to_s}.concat"] = concatenated_json
+    end
+
+    def render_models(content, full_path, options)
+      
+      filename, subdirectory = full_path_to_parts(full_path)
+
+      @described_models = []
+      @json_models = {}
+      @fragment_meta = {}
+      enabled = true
+
+      content.split(HTML_COMMENT_REGEX).each do |text|
+        dsl_command = HTML_COMMENT_REGEX =~ text
+
+        if dsl_command 
+          enabled = false if disable_dsl(text)
+          enabled = true if enable_dsl(text)
+        end
+
+        if dsl_command && enabled
+          extract_model(text, filename, subdirectory)
+          extract_fragment_meta(text, subdirectory)
+          extract_attribute(text)
+          pop_attribute(text)
+          pop_model(text)
+        else
+          append_to_attribute(text) 
+        end
+      end
+
+      @described_models.each do |model|
+        finalize_attribute(model)
+      end
+
+      @json_models.dup
+    end
+
+    def add_content_fragment_for_file(json, full_path, concatenated, meta)
+      content_fragment = fragment_attributes_from_path(full_path).merge("meta" => meta, "concatenated" => concatenated)
+
+      json["markdown_record/content_fragment"] ||= []
+      json["markdown_record/content_fragment"] << content_fragment
+    end
+
+    def concatenate_json_recursively(directory_hash, concatenated_json)
+      directory_hash.each do |key, value|
+        case value.class.name
+        when Hash.name
+          concatenate_json_recursively(value, concatenated_json)
+        when Array.name
+          concatenated_json[key] ||= []
+          concatenated_json[key] += value
+        end
+      end
+    end
+
+    def save_content_recursively(content, options, rendered_subdirectory)
+      if content&.values&.first.is_a?(Array)
+        fragments = content.slice("markdown_record/content_fragment")
+        non_fragments = content.except("markdown_record/content_fragment")
+        path = clean_path(rendered_subdirectory.to_s)
+        @file_saver.save_to_file(non_fragments.to_json, "#{path}.json", options)
+        @file_saver.save_to_file(fragments.to_json, "#{path}.json", options, true)
+      else
+        content.each do |key, value|
+          child_path = rendered_subdirectory.join(key)
+          save_content_recursively(value, options, child_path)
+        end
+      end
+    end
+
+    def extract_model(html, filename, subdirectory)
+      model = model_dsl(html)
+
+      if model
+        return unless model["type"].present?
+
+        model["subdirectory"] = clean_path(subdirectory)
+        model["filename"] = clean_path(filename)
+
+        # reset "type" to not have a prefix, in order to ensure
+        # consistent results between the internal only :klass filter
+        # which is used as an index in a hash (and has the prefix removed 
+        # for consistency and deterministic behavior)
+        # and the externally filterable :type field.
+        model["type"] = model["type"].delete_prefix("::").strip
+        
+        @json_models[model["type"]] ||= []
+        @json_models[model["type"]] << model
+  
+        @described_models.push(model)
+      end
+    end
+
+    def extract_fragment_meta(html, subdirectory)
+      meta = fragment_dsl(html)
+
+      if meta
+        @fragment_meta = meta
+      end
+
+      directory_meta = directory_fragment_dsl(html)
+
+      if directory_meta
+        @directory_meta[subdirectory] = directory_meta
+      end
+    end
+
+    def extract_attribute(text)
+      attribute, type = attribute_dsl(text)
+      type ||= "md"
+
+      if @described_models.last && attribute
+        model = @described_models.last
+        model[:described_attribute_type] = type
+        model[:described_attribute] = attribute
+        model[model[:described_attribute]] = []
+      end
+    end
+
+    def append_to_attribute(text)
+      return if text.match(HTML_COMMENT_REGEX)
+
+      @described_models.each do |model|
+        next unless model[:described_attribute].present?
+
+        model[model[:described_attribute]] ||= []
+        model[model[:described_attribute]] << text
+      end
+    end
+
+    def pop_attribute(text)
+      if @described_models.last && end_attribute_dsl(text)
+        finalize_attribute(@described_models.last)
+      end
+    end
+
+    def pop_model(html)      
+      if end_model_dsl(html)
+        @described_models.pop
+      end
+    end
+
+    def finalize_attribute(model)
+      return if model[:described_attribute].nil?
+
+      attribute_name = model.delete(:described_attribute)
+      attribute = model[attribute_name].join("\n")
+      type = model.delete(:described_attribute_type)
+
+      case type
+      when "html"
+        renderer = Redcarpet::Render::HTML.new
+        attribute = Redcarpet::Markdown.new(renderer).render(attribute)
+      when "md"
+        attribute = attribute.strip
+      when "int"
+        attribute = attribute.strip.gsub("\n", "").to_i
+      when "float"
+        attribute = attribute.strip.gsub("\n", "").to_f
+      when "string"
+        attribute = Redcarpet::Markdown.new(::Redcarpet::Render::StripDown).render(attribute).strip
+      end
+
+      model[attribute_name] = attribute
+    end
+  end
+end

data/lib/markdown_record/model_inflator.rb

@@ -0,0 +1,107 @@
+require "markdown_record/path_utilities"
+
+module MarkdownRecord
+  class ModelInflator
+    include ::MarkdownRecord::PathUtilities
+    
+    def initialize(source_models = nil)
+      @indexer = ::MarkdownRecord::Indexer.new
+    end
+
+    def where(filters, subdirectory = nil)
+      path = if subdirectory.nil?
+               file_path = base_rendered_path
+               file_path = "#{file_path}_fragments" if filters[:klass] == ::MarkdownRecord::ContentFragment
+               "#{file_path}.json"
+             else
+               file_path = subdirectory
+               file_path = "#{file_path}_fragments" if filters[:klass] == ::MarkdownRecord::ContentFragment
+               "#{base_rendered_path}/#{file_path}.json"
+             end
+
+      json = json_source(path)
+      json.delete(::MarkdownRecord::ContentFragment.json_klass) if filters.delete(:exclude_fragments)
+      json = filters[:klass].present? ? json[filters.delete(:klass).json_klass] : json.values.flatten
+      json ||= []
+
+      filtered_models = json.select do |model|
+        passes_filters?(model.with_indifferent_access, filters.dup)
+      end
+
+      filtered_models.map do |model|
+        model["type"].camelize.safe_constantize&.new(model)
+      end
+    end
+
+    def json_source(path)
+      if Pathname.new(path).exist?
+        json = JSON.parse(File.read(path))
+        return json
+      end
+
+      json = ::MarkdownRecord::JsonRenderer.new.render_models_for_subdirectory(subdirectory: "",:concat => true, :deep => true, :save => false, :render_content_fragment_json => true)
+      
+      filename, subdirectory = full_path_to_parts(path)
+
+      tokens = subdirectory.gsub(base_rendered_root, "").split("/")
+      tokens << "#{filename}.concat"
+      json.dig(*tokens) || {}
+    end
+
+    def passes_filters?(attributes, filters)
+      passes = true
+
+      not_filters = filters.delete(:__not__)
+      or_filters = filters.delete(:__or__)
+      and_filters = filters.delete(:__and__)
+
+      filters.each do |key, value|
+        passes &&= passes_filter?(attributes, key, value)
+      end
+      
+      not_filters&.each do |key, value|    
+        passes &&= !passes_filter?(attributes, key, value)
+      end
+
+      or_temp = !or_filters&.any?
+      or_filters&.each do |sub_filter|
+        or_temp ||= passes_filters?(attributes, sub_filter.dup)
+      end
+
+      and_temp = true
+      and_filters&.each do |sub_filter|
+        and_temp &&= passes_filters?(attributes, sub_filter.dup)
+      end
+
+      passes &&= or_temp
+      passes &&= and_temp
+      passes
+    end
+
+    def passes_filter?(attributes, filter_key, filter_value)
+      case filter_value.class.name
+      when Array.name
+        filter_value.include?(attributes[filter_key])
+      when Symbol.name
+        if filter_value == :not_null
+          !attributes[filter_key].nil?
+        elsif filter_value == :null
+          attributes[filter_key].nil?
+        else
+          false
+        end
+      when Hash.name
+        if attributes[filter_key]&.class == ActiveSupport::HashWithIndifferentAccess
+          passes_filters?(attributes[filter_key], filter_value)
+        end
+        
+      when nil.class.name
+        attributes[filter_key].nil?
+      when Regexp.name
+        attributes[filter_key] =~ filter_value
+      else
+        filter_value == attributes[filter_key]
+      end
+    end
+  end
+end

data/lib/markdown_record/path_utilities.rb

@@ -0,0 +1,86 @@
+
+
+module MarkdownRecord
+  module PathUtilities
+
+    def full_path_to_parts(full_path)
+      rendered_path = Pathname.new("/").join(full_path)
+      
+      filename = clean_path(rendered_path.basename)
+      filename = remove_numeric_prefixes(filename)
+      
+      subdirectory = clean_path(rendered_path.parent)
+      subdirectory = remove_numeric_prefixes(subdirectory)
+
+      [filename, subdirectory, full_path.to_s.split(".").last]
+    end
+
+    def rendered_path(full_path)
+      rendered_path = clean_path(full_path.to_s)
+      rendered_path = remove_numeric_prefixes(rendered_path)
+      Pathname.new(rendered_path)
+    end
+
+    def path_to_fragment_id(full_path)
+      rendered_path(full_path).to_s
+    end
+
+    def base_content_path
+      basename = ::MarkdownRecord.config.content_root.basename
+
+      # must use "/" so that `.parent` returns correctly
+      # for top level paths.
+      Pathname.new("/").join(basename)
+    end
+
+    def erb_locals_from_path(full_path)
+      filename, subdirectory = *full_path_to_parts(full_path)
+      
+      frag_id = path_to_fragment_id(full_path)
+
+      fragment = ::MarkdownRecord::ContentFragment.find(frag_id)
+      
+      {
+        filename: filename, 
+        subdirectory: subdirectory, 
+        frag_id: frag_id,
+        fragment: fragment
+      }
+    end
+
+    def fragment_attributes_from_path(full_path)
+      filename, subdirectory = *full_path_to_parts(full_path)
+
+      frag_id = path_to_fragment_id(full_path)
+      
+      {
+        id: frag_id,
+        type: MarkdownRecord::ContentFragment.name.underscore,
+        subdirectory: subdirectory,
+        filename: filename
+      }.stringify_keys
+    end
+
+    def clean_path(path)
+      path.to_s.gsub("_fragments", "").gsub(/(\.concat|\.md\.erb|\.md|\.json|\.html)/,"").delete_prefix("/")
+    end
+
+    def base_rendered_path
+      ::MarkdownRecord.config.rendered_content_root.join(::MarkdownRecord.config.content_root.basename)
+    end
+
+    def base_rendered_root
+      clean_path(::MarkdownRecord.config.rendered_content_root.to_s)
+    end
+
+    def remove_numeric_prefixes(filename_or_id)
+      if MarkdownRecord.config.ignore_numeric_prefix
+        parts = filename_or_id.split("/")
+        parts = parts.map { |p| p.gsub(/^\d+_/,"")}
+        parts.join("/")
+      else
+        filename_or_id
+      end
+    end
+  end
+end

data/lib/markdown_record/rendering.rb

@@ -0,0 +1,57 @@
+module MarkdownRecord
+  module Rendering
+    def generate_render_strategy_options(options)
+
+      strategy_options = if options[:strat].present?
+                           ::MarkdownRecord.config.render_strategy_options(options[:strat].to_sym)
+                         else
+                           ::MarkdownRecord.config.render_strategy_options
+                         end
+
+      strategy_options.merge!(:save => options[:save])
+      strategy_options[:render_content_fragment_json] = options[:frag] || ::MarkdownRecord.config.render_content_fragment_json
+      strategy_options
+    end
+
+    def report_rendered_files(lines, file_saver)
+      file_count = (file_saver.saved_files[:json].count + file_saver.saved_files[:html].count)
+      
+      record_and_save lines, "---------------------------------------------------------------"
+      file_saver.saved_files[:json].each do |file_name|
+        record_and_save lines, "rendered: #{file_name}", report_line_color(options[:save])
+      end
+      file_saver.saved_files[:html].each do |file_name|
+        record_and_save lines, "rendered: #{file_name}", report_line_color(options[:save])
+      end
+      record_and_save lines, "---------------------------------------------------------------"
+      record_and_save lines, "#{file_count} files rendered."
+      record_and_save lines, "#{options[:save] ? file_count : 0} files saved."
+
+      ::MarkdownRecord.config.rendered_content_root.join("rendered.txt").open('wb') do |file|
+        file << lines.join("\n")
+      end
+    end
+
+    def report_start(lines, strategy_options, formats)
+      record_and_save lines, "---------------------------------------------------------------"
+      record_and_save lines, "rendering #{formats} content with options #{strategy_options} ..."
+    end
+
+    def report_line_color(saved)
+      color = saved ? :green : :yellow
+    end
+
+    def record_and_save(lines, text, color = nil)
+      lines << text
+      say text, color
+    end
+
+    def validate
+      MarkdownRecord::Validator.new.validate
+      true
+    rescue MarkdownRecord::Errors::Base => e
+      say e.message, :red
+      false
+    end
+  end
+end

data/lib/markdown_record/validator.rb

@@ -0,0 +1,59 @@
+require "markdown_record/errors/duplicate_filename_error"
+require "markdown_record/errors/duplicate_id_error"
+require "markdown_record/errors/missing_parent_error"
+
+module MarkdownRecord
+  class Validator
+    include ::MarkdownRecord::PathUtilities
+
+    def initialize
+      @index = ::MarkdownRecord::Indexer.new.index
+      @json = ::MarkdownRecord::JsonRenderer.new.render_models_for_subdirectory(subdirectory: "",:concat => true, :deep => true, :save => false, :render_content_fragment_json => true)
+    end
+
+    def validate
+      validate_filenames(@index)
+      validate_models
+      validate_fragments
+    end
+
+    def validate_filenames(val)
+      if val.is_a?(Hash)
+        temp_keys = val.keys.map { |v| remove_numeric_prefixes(v) }
+        
+        dups = temp_keys.group_by{|e| clean_path(e)}.keep_if{|_, e| e.length > 1}
+        if dups.any?
+          raise ::MarkdownRecord::Errors::DuplicateFilenameError.new(dups.keys)
+        else
+          val.each do |k, v|
+            validate_filenames(v)
+          end
+        end
+      end
+    end
+
+    def validate_models
+      @json["#{base_content_path.basename}.concat"].each do |klass, array|
+        ids = array.map { |o| o["id"] }
+        dups = ids.group_by{|e| e}.keep_if{|_, e| e.length > 1}
+        if dups.any?
+          raise ::MarkdownRecord::Errors::DuplicateIdError.new(klass, dups.keys.first)
+        end
+      end
+    end
+
+    def validate_fragments
+      frags = @json["#{base_content_path.basename}.concat"]["markdown_record/content_fragment"]
+      frags.each do |frag|
+        parent_id = frag.dig("meta", "parent_id")
+        if parent_id
+          unless frags.any? { |f| f["id"] == parent_id}
+            raise ::MarkdownRecord::Errors::MissingParentError.new(parent_id)
+          end
+        end
+      end
+    end
+
+
+  end
+end

data/lib/markdown_record/version.rb

@@ -0,0 +1,3 @@
+module MarkdownRecord
+  VERSION = "0.1.3"
+end

data/lib/markdown_record.rb

@@ -0,0 +1,28 @@
+require "markdown_record/version"
+require "markdown_record/engine"
+require "markdown_record/configuration"
+require "markdown_record/path_utilities"
+require "markdown_record/validator"
+require "markdown_record/rendering"
+require "markdown_record/cli"
+require "markdown_record/content_dsl"
+require "markdown_record/indexer"
+require "markdown_record/file_saver"
+require "markdown_record/html_renderer"
+require "markdown_record/json_renderer"
+require "markdown_record/model_inflator"
+require "markdown_record/association"
+require "markdown_record/associations"
+require "markdown_record/content_associations"
+require "markdown_record/base"
+require "markdown_record/content_fragment"
+
+module MarkdownRecord
+  def self.config
+    Configuration.instance
+  end
+
+  def self.configure
+    yield(Configuration.instance)
+  end
+end

data/templates/Thorfile

@@ -0,0 +1,3 @@
+require File.expand_path("../config/environment", __FILE__)
+
+Dir["./lib/tasks/**/*.thor"].sort.each { |f| load f }

data/templates/base/content/example_content.md

@@ -0,0 +1,3 @@
+# Example Content
+
+This directory is where your markdown source files will live.

data/templates/base/layouts/_concatenated_layout.html.erb

@@ -0,0 +1,8 @@
+<style>
+
+</style>
+
+<div class="concatenated">
+  <%= raw(html) %>
+  <%= raw("<!--#{subdirectory}/#{filename}-->") %>
+</div>

data/templates/base/layouts/_custom_layout.html.erb

@@ -0,0 +1,7 @@
+<style>
+
+</style>
+<div>
+  <%= raw(html) %>
+  <%= raw("<!--#{subdirectory}/#{filename}-->") %>
+</div>

data/templates/base/layouts/_file_layout.html.erb

@@ -0,0 +1,8 @@
+<style>
+
+</style>
+
+<div class="file">
+  <%= raw(html) %>
+  <%= raw("<!--#{subdirectory}/#{filename}-->") %>
+</div>

data/templates/base/layouts/_global_layout.html.erb

@@ -0,0 +1,8 @@
+<style>
+
+</style>
+
+<div class="container">
+  <%= raw(html) %>
+  <%= raw("<!--#{subdirectory}/#{filename}-->") %>
+</div>