yarrow 0.8.6 → 0.9.0

data/lib/yarrow/content/expansion/traversal.rb

@@ -0,0 +1,67 @@
+module Yarrow
+  module Content
+    module Expansion
+      class Traversal
+        attr_reader :graph, :policy, :aggregator
+
+        def initialize(graph, policy)
+          @graph = graph
+          @policy = policy
+          @aggregator = policy.aggregator_const.new(graph)
+        end
+
+        # If source path represents entire content dir, then include the entire
+        # content dir instead of scanning from a subfolder matching the name of
+        # the collection.
+        def source_node
+          if policy.source_path == "."
+            graph.n(:root).out(:directory)
+          else
+            graph.n(name: policy.source_path)
+          end
+        end
+
+        def on_root_visited(root_node)
+          aggregator.expand_container(root_node, policy)
+        end
+
+        def on_directory_visited(dir_node)
+          # TODO: match on potential directory extension/filter
+          aggregator.expand_collection(dir_node, policy)
+        end
+
+        def on_file_visited(file_node)
+          # TODO: dispatch underscore prefix or index files separately
+          # TODO: match on file extension
+          aggregator.expand_entity(file_node, policy)
+        end
+
+        def on_traversal_initiated
+          aggregator.before_traversal(policy)
+        end
+
+        def on_traversal_completed
+          aggregator.after_traversal(policy)
+        end
+
+        def expand
+          on_traversal_initiated
+
+          traversal = source_node.depth_first.each
+          
+          on_root_visited(traversal.next)
+
+          loop do
+            node = traversal.next
+            case node.label
+            when :directory then on_directory_visited(node)
+            when :file then on_file_visited(node)
+            end
+          end
+
+          on_traversal_completed
+        end
+      end
+    end
+  end
+end

data/lib/yarrow/content/expansion/tree.rb

@@ -3,98 +3,35 @@ module Yarrow
     module Expansion
       class Tree < Strategy
         def expand(policy)
-          #p graph.n(:root).out(:directory).to_a.count
-          #policy.match()
-
-          #p graph.n(:root).out(:directory).first.props[:name]
-          type = policy.container
-
           # If match path represents entire content dir, then include the entire
           # content dir instead of scanning from a subfolder matching the name of
           # the collection.
           #start_node = if policy.match_path == "."
-          start_node = if true
+          start_node = if policy.match_path == "."
             # TODO: match against source_dir
             graph.n(:root).out(:directory)
           else
-            graph.n(:root).out(name: policy.container.to_s)
+            graph.n(name: policy.match_path)
           end
 
-          # Extract metadata from given start node
-          #collection_metadata = extract_metadata(start_node, policy.container)
-
           # Collect all nested collections in the subgraph for this content type
-          subcollections = {}
-          entity_links = []
-          index_links = []
-          index = nil
+          @subcollections = {}
+          @entity_links = []
+          @index_links = []
+          @index = nil
 
           # Scan and collect all nested files from the root
           start_node.depth_first.each do |node|
             if node.label == :directory
-              # Create a collection node representing a collection of documents
-              index = graph.create_node do |collection_node|
-
-                collection_attrs = {
-                  name: node.props[:name],
-                  title: node.props[:name].capitalize,
-                  body: ""
-                }
-
-                populate_collection(collection_node, policy, collection_attrs)
-              end
-
-              # Add this collection id to the lookup table for edge construction
-              subcollections[node.props[:path]] = index
-
-              # Join the collection to its parent
-              unless node.props[:slug] == type.to_s || !subcollections.key?(node.props[:entry].parent.to_s)
-                graph.create_edge do |edge|
-                  edge.label = :child
-                  edge.from = subcollections[node.props[:entry].parent.to_s].id
-                  edge.to = index.id
-                end
-              end
+              expand_directory(policy, node)
             elsif node.label == :file
-              body, meta = process_content(node.props[:entry])
-              meta = {} if !meta
-
-              # TODO: document mapping convention for index pages and collection metadata
-              # TODO: underscore _index pattern?
-              bare_basename = node.props[:entry].basename(node.props[:entry].extname)
-              if bare_basename.to_s == "index"
-                index_links << {
-                  parent_id: subcollections[node.props[:entry].parent.to_s],
-                  index_attrs: meta.merge({ body: body})
-                }
-              else
-                # Create an entity node representing a file mapped to a unique content object
-                entity = graph.create_node do |entity_node|
-
-                  entity_slug = node.props[:entry].basename(node.props[:entry].extname).to_s
-
-                  entity_attrs = {
-                    name: entity_slug,
-                    title: entity_slug.gsub("-", " ").capitalize,
-                    body: body
-                  }
-
-                  populate_entity(entity_node, policy, entity_attrs.merge(meta || {}))
-                end
-
-                # We may not have an expanded node for the parent collection if this is a
-                # preorder traversal so save it for later
-                entity_links << {
-                  parent_id: subcollections[node.props[:entry].parent.to_s],
-                  child_id: entity
-                }
-              end
+              expand_file_by_extension(policy, node)
             end
           end
 
           # Once all files and directories have been expanded, connect all the child
           # edges between collections and entities
-          entity_links.each do |entity_link|
+          @entity_links.each do |entity_link|
             graph.create_edge do |edge|
               edge.label = :child
               edge.from = entity_link[:parent_id].id
@@ -103,10 +40,53 @@ module Yarrow
           end
 
           # Merge index page body and metadata with their parent collections
-          index_links.each do |index_link|
+          @index_links.each do |index_link|
             merge_collection_index(index_link[:parent_id], policy, index_link[:index_attrs])
           end
         end
+
+        def expand_file_by_basename(policy, node)
+          body, meta = process_content(node.props[:entry])
+          meta = {} if !meta
+
+
+        end
+
+        def expand_file_by_extension(policy, node)
+          body, meta = process_content(node.props[:entry])
+          meta = {} if !meta
+
+          # TODO: document mapping convention for index pages and collection metadata
+          # TODO: underscore _index pattern?
+          bare_basename = node.props[:entry].basename(node.props[:entry].extname)
+          if bare_basename.to_s == "index"
+            @index_links << {
+              parent_id: @subcollections[node.props[:entry].parent.to_s],
+              index_attrs: meta.merge({ body: body})
+            }
+          else
+            # Create an entity node representing a file mapped to a unique content object
+            entity = graph.create_node do |entity_node|
+
+              entity_slug = node.props[:entry].basename(node.props[:entry].extname).to_s
+
+              entity_attrs = {
+                name: entity_slug,
+                title: entity_slug.gsub("-", " ").capitalize,
+                body: body
+              }
+
+              populate_entity(entity_node, policy, entity_attrs.merge(meta || {}))
+            end
+
+            # We may not have an expanded node for the parent collection if this is a
+            # preorder traversal so save it for later
+            @entity_links << {
+              parent_id: @subcollections[node.props[:entry].parent.to_s],
+              child_id: entity
+            }
+          end
+        end
       end
     end
   end

data/lib/yarrow/content/model.rb

@@ -14,8 +14,9 @@ module Yarrow
 
       def expand(graph)
         @policies.each_value do |policy|
-          strategy = Expansion::Tree.new(graph)
-          strategy.expand(policy)
+          #strategy = policy.expansion_strategy.new(graph)
+          traversal = Expansion::Traversal.new(graph, policy)
+          traversal.expand
         end
       end
 

data/lib/yarrow/content/policy.rb

@@ -1,13 +1,11 @@
 module Yarrow
   module Content
     class Policy
-      DEFAULT_HOME_NESTING = false
-
-      DEFAULT_EXPANSION = :tree
+      DEFAULT_EXPANSION = :filename_map
 
       DEFAULT_EXTENSIONS = [".md", ".yml", ".htm"]
 
-      DEFAULT_MATCH_PATH = "."
+      DEFAULT_SOURCE_PATH = "."
 
       MODULE_SEPARATOR = "::"
 
@@ -15,15 +13,35 @@ module Yarrow
       def self.from_spec(policy_label, policy_props, module_prefix="")
         # TODO: validate length, structure etc
 
-        # If the spec holds a symbol value then treat it as a container => entity mapping
+        # If the spec holds a symbol value then treat it as an entity mapping
         if policy_props.is_a?(Symbol)
-          new(policy_label, policy_props, DEFAULT_EXPANSION, DEFAULT_EXTENSIONS, DEFAULT_MATCH_PATH, module_prefix)
-
-        # Otherwise scan through all the props and fill in any gaps
+          new(
+            policy_label,
+            policy_label,
+            policy_props,
+            DEFAULT_EXPANSION,
+            DEFAULT_EXTENSIONS,
+            policy_label.to_s,
+            module_prefix
+          )
+
+        # If the spec holds a string value then treat it as a source path mapping
+        elsif policy_props.is_a?(String)
+          new(
+            policy_label,
+            policy_label,
+            Yarrow::Symbols.to_singular(policy_label),
+            DEFAULT_EXPANSION,
+            DEFAULT_EXTENSIONS,
+            policy_props,
+            module_prefix
+          )
+
+        # Otherwise scan through the spec and fill in any gaps
         else
-          # Use explicit container name if provided
-          container = if policy_props.key?(:container)
-            policy_props[:container]
+          # Use explicit collection name if provided
+          collection = if policy_props.key?(:collection)
+            policy_props[:collection]
           else
             # If an entity name is provided use its plural for the container name
             if policy_props.key?(:entity)
@@ -33,12 +51,19 @@ module Yarrow
             end
           end
 
+          # Use explicit container name if provided
+          container = if policy_props.key?(:container)
+            policy_props[:container]
+          else
+            collection
+          end
+
           # Use explicit entity name if provided
           entity = if policy_props.key?(:entity)
             policy_props[:entity]
           else
-            if policy_props.key?(:container)
-              Yarrow::Symbols.to_singular(policy_props[:container])
+            if policy_props.key?(:collection)
+              Yarrow::Symbols.to_singular(policy_props[:collection])
             else
               Yarrow::Symbols.to_singular(policy_label)
             end
@@ -58,22 +83,35 @@ module Yarrow
             DEFAULT_EXTENSIONS
           end
 
-          # TODO: handle this in expansion strategies
-          match_path = DEFAULT_MATCH_PATH
+          # If match path is provided, treat it as a basename
+          source_path = if policy_props.key?(:source_path)
+            policy_props[:source_path]
+          else
+            DEFAULT_SOURCE_PATH
+          end
 
           # Construct the new policy
-          new(container, entity, expansion, extensions, match_path, module_prefix)
+          new(
+            container,
+            collection,
+            entity,
+            expansion,
+            extensions,
+            source_path,
+            module_prefix
+          )
         end
       end
 
-      attr_reader :container, :entity, :expansion, :extensions, :match_path, :module_prefix
+      attr_reader :container, :collection, :entity, :expansion, :extensions, :source_path, :module_prefix
 
-      def initialize(container, entity, expansion, extensions, match_path, module_prefix)
+      def initialize(container, collection, entity, expansion, extensions, source_path, module_prefix)
         @container = container
+        @collection = collection
         @entity = entity
         @expansion = expansion
         @extensions = extensions
-        @match_path = match_path
+        @source_path = source_path
         @module_prefix = module_prefix.split(MODULE_SEPARATOR)
       end
 
@@ -81,12 +119,30 @@ module Yarrow
         @container_const ||= Yarrow::Symbols.to_module_const([*module_prefix, container])
       end
 
-      alias_method :collection, :container
-      alias_method :collection_const, :container_const
+      def collection_const
+        begin
+          @collection_const ||= Yarrow::Symbols.to_module_const([*module_prefix, collection])
+        rescue NameError
+          raise NameError, "cannot map undefined entity `#{collection}`"
+        end
+      end
 
       def entity_const
         @entity_const ||= Yarrow::Symbols.to_module_const([*module_prefix, entity])
       end
+
+      def aggregator_const
+        case expansion
+        when :filename_map then Expansion::FilenameMap
+        when :directory_merge then Expansion::DirectoryMerge
+        else
+          raise "No match strategy exists for :#{expansion}"
+        end
+      end
+
+      def match_by_extension(candidate)
+        extensions.include?(candidate)
+      end
     end
   end
 end

data/lib/yarrow/content/source.rb

@@ -15,6 +15,7 @@ module Yarrow
             dir.label = :directory
             dir.props = {
               name: root_dir_entry.basename.to_s,
+              basename: root_dir_entry.basename.to_s,
               path: root_dir_entry.to_s,
               entry: root_dir_entry
             }
@@ -30,6 +31,7 @@ module Yarrow
             root_dir_entry.to_s => root_dir.id
           }
 
+          # TODO: merge entry and path props
           Pathname.glob("#{input_dir}/**/**").each do |entry|
             if entry.directory?
               content_node = create_node do |dir|
@@ -40,6 +42,7 @@ module Yarrow
                 # dir.props[:entry] = entry
                 dir.props = {
                   name: entry.basename.to_s,
+                  basename: entry.basename.to_s,
                   path: entry.to_s,
                   entry: entry
                 }
@@ -56,6 +59,7 @@ module Yarrow
 
                 file.props = {
                   name: entry.basename.to_s,
+                  basename: entry.basename.sub_ext('').to_s,
                   ext: entry.extname.to_s,
                   path: entry.to_s,
                   entry: entry

data/lib/yarrow/format/markdown.rb

@@ -0,0 +1,67 @@
+module Yarrow
+  module Format
+    class Markdown < ContentType[".md", ".markdown"]
+      include Methods::FrontMatter
+
+      def initialize(source)
+        @source = source.to_s
+        @document = Kramdown::Document.new(@source)
+      end
+
+      def to_s
+        @source
+      end
+  
+      def to_dom
+        @document.root
+      end
+  
+      def to_html
+        @document.to_html
+      end
+  
+      def links
+        @links ||= select_links
+      end
+  
+      def title
+        @title ||= select_title
+      end
+  
+      private
+  
+      def select_links
+        stack = to_dom.children
+        hrefs = [] # TODO: distinguish between internal and external
+  
+        while !stack.empty?
+          next_el = stack.pop
+  
+          if next_el.type == :a
+            hrefs << next_el.attr["href"]
+          else
+            stack.concat(next_el.children) if next_el.children
+          end
+        end
+  
+        hrefs.reverse
+      end
+  
+      def select_title
+        stack = to_dom.children
+  
+        while !stack.empty?
+          next_el = stack.pop
+  
+          if next_el.type == :header and next_el.options[:level] == 1
+            return next_el.options[:raw_text]
+          else
+            stack.concat(next_el.children) if next_el.children
+          end
+        end
+  
+        nil
+      end
+    end
+  end
+end

data/lib/yarrow/format/methods/front_matter.rb

@@ -0,0 +1,58 @@
+module Yarrow
+  module Format
+    module Methods
+      # Utility methods for working with text formats containing frontmatter separators.
+      module FrontMatter
+        module ClassMethods
+          def read(path)
+            text = File.read(path, :encoding => "utf-8")
+            source, metadata = read_yfm(path)
+            Yarrow::Format::Contents.new(new(source), metadata)
+          end
+
+          def read_yfm(path)
+            text = File.read(path, :encoding => 'utf-8')
+            extract_yfm(text, symbolize_keys: true)
+          end
+  
+          def extract_yfm(text, options={})
+            pattern = /^(---\s*\n.*?\n?)^(---\s*$\n?)/m
+            if text =~ pattern
+              content = text.sub(pattern, "")
+  
+              begin
+                if options.key?(:symbolize_keys)
+                  meta = YAML.load($1, symbolize_names: true)
+                else
+                  meta = YAML.load($1)
+                end
+                return [content, meta]
+              rescue Psych::SyntaxError => error
+                if defined? ::Logger
+                  # todo: application wide logger
+                  #logger = ::Logger.new(STDOUT)
+                  #logger.error "#{error.message}"
+                end
+                return [content, nil]
+              end
+            end
+  
+            [text, nil]
+          end
+  
+          def write_yfm(name, text, meta)
+            # Symbolized keys are better to deal with when manipulating data in
+            # Ruby but are an interop nightmare when serialized so here we do a
+            # round-trip through JSON encoding to ensure all keys are string
+            # encoded before dumping them to the front matter format.
+            File.write(name, [YAML.dump(meta.to_json), "---", text].join("\n"))
+          end
+        end
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+      end
+    end
+  end
+end

data/lib/yarrow/format/methods/metadata.rb

@@ -0,0 +1,22 @@
+module Yarrow
+  module Format
+    module Methods
+      module Metadata
+        module ClassMethods
+          def read(path)
+            text = File.read(path, :encoding => "utf-8")
+            Yarrow::Format::Contents.new(nil, parse(text))
+          end
+
+          def parse(text)
+            new(text)
+          end
+        end
+
+        def self.included(base)
+          base.extend(ClassMethods)
+        end
+      end
+    end
+  end
+end

data/lib/yarrow/format/yaml.rb

@@ -0,0 +1,19 @@
+module Yarrow
+  module Format
+    class Yaml < ContentType[".yml", ".yaml"]
+      include Methods::Metadata
+
+      def initialize(source)
+        @data = YAML.load(source, symbolize_names: true)
+      end
+
+      def [](key)
+        @data[key]
+      end
+
+      def to_h
+        @data
+      end
+    end
+  end
+end

data/lib/yarrow/format.rb

@@ -0,0 +1,71 @@
+module Yarrow
+  module Format
+    class Contents
+      attr_reader :document, :metadata
+
+      def initialize(document, metadata)
+        @document = document
+        @metadata = metadata
+      end
+    end
+
+    Registry = {}
+
+    class ContentType
+      def self.[](*extensions)
+        @@extensions_cache = extensions
+        self
+      end
+
+      def self.inherited(media_type)
+        return if @@extensions_cache.nil?
+
+        @@extensions_cache.each do |extname|
+          Registry[extname] = media_type
+        end
+
+        @@extensions_cache = nil
+      end
+
+      def initialize(source)
+        @source = source
+      end
+
+      def to_s
+        @source
+      end
+    end
+
+    # Pass in a source path and get back a parsed representation of the
+    # content if it is in a known text format. Mostly used as a fallback if
+    # a custom parser or processing chain is not configured for a content
+    # type.
+    def self.read(name)
+      path = if name.is_a?(Pathname)
+        name
+      else
+        Pathname.new(name)
+      end
+
+      # case path.extname
+      # when '.htm', '.md', '.txt', '.yfm'
+      #   Markdown.read(path)
+      # when '.yml'
+      #   [nil, YAML.load(File.read(path.to_s), symbolize_names: true)]
+      # when '.json'
+      #   [nil, JSON.parse(File.read(path.to_s))]
+      # end
+    
+      unless Registry.key?(path.extname)
+        raise "Unsupported format: #{path.extname} (#{path})"
+      end
+        
+      Registry[path.extname].read(path)
+    end
+  end
+end
+
+require_relative "format/methods/front_matter"
+require_relative "format/methods/metadata"
+require_relative "format/markdown"
+require_relative "format/yaml"