yarrow 0.5.0 → 0.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7360a9c100d2a45cbf57a77fcec6a02f21156a6af46d22a6c438dd43ab07557f
-  data.tar.gz: c71c2e48c2b3118b1d3d734430925f9bee7981a89462df9a8e05bfc5d12fc3c7
+  metadata.gz: 4be44c931a92757c5be67d594977711597a71560b8ea37f6ea666842c128ddd8
+  data.tar.gz: c8af0e56a102cec9b111d921a5333c703bd341084ebd839fb4eb870728530dff
 SHA512:
-  metadata.gz: 77ed4fe864b0ad9f7d4a73b9787619f166eb5aa2631c70377e57de4c1f781f8da2bca717dafd98a940aff2e637590335e0076894b99a2f4e7ebbf94151acaa56
-  data.tar.gz: 6732e24c0275884b3cfbab5a417ab755b900ca8d94ffa00d7a84a6c90a042325615221785085d1913f1a0b26b54399b500d90f5beb6b6cd1c7183236e12beade
+  metadata.gz: 265f331b400a0ddcfa6163da67c7fb30c57ee4c3506fa2ae8be4e346512884134dfcc8cfc9ee0ffd7e55575d7a7f3433d3c3f1b1f8fc172939f3423dac92c154
+  data.tar.gz: 4ea5214183ac720a1cec31f9ac6b6371012c2ac79476278afc2bb55dcdcf6155c8e6688ab03c349a2035aa62c6ef6e5ca0b0ae75318975d590274a1f1a4913b3

data/.github/workflows/ruby.yml

@@ -0,0 +1,23 @@
+name: ruby
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        ruby: [2.7, '3.0', jruby-head]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run tests
+      run: bundle exec rspec

data/.gitignore

@@ -4,4 +4,5 @@ fabfile.pyc
 pirum.xml
 Gemfile.lock
 yarrow-*.gem
-coverage
+coverage
+bin/scripts

data/README.md

@@ -1,21 +1,15 @@
 Yarrow
 ======
 
-[![Gem Version](http://img.shields.io/gem/v/yarrow.svg)][gem]
-[![Build Status](http://img.shields.io/travis/maetl/yarrow.svg)][travis]
-[![Dependency Status](http://img.shields.io/gemnasium/maetl/yarrow.svg)][gemnasium]
-[![Code Climate](http://img.shields.io/codeclimate/github/maetl/yarrow.svg)][codeclimate]
-[![Coverage Status](http://img.shields.io/coveralls/maetl/yarrow.svg)][coveralls]
+[![Gem Version](https://img.shields.io/gem/v/yarrow.svg)][gem]
+[![Build Status](https://img.shields.io/github/workflow/status/maetl/yarrow/ruby/main)][github]
 
 [gem]: https://rubygems.org/gems/yarrow
-[travis]: https://travis-ci.org/maetl/yarrow
-[gemnasium]: https://gemnasium.com/maetl/yarrow
-[codeclimate]: https://codeclimate.com/github/maetl/yarrow
-[coveralls]: https://coveralls.io/r/maetl/yarrow
+[github]: https://github.com/maetl/yarrow
 
-Yarrow is a tool for generating well structured documentation from a variety of input sources.
+Yarrow is a framework for generating well structured publishing outputs from a variety of input sources.
 
-Unlike most static site generators and code documentation tools, Yarrow is written with design and content-strategy in mind. It does not impose its own structure on your content. This makes it appropriate for building static sites and blogs as well as style guides and API docs.
+Unlike most static site generators and code documentation tools, Yarrow is written with design and content-strategy in mind. It does not impose its own structure on your content. This makes it useful for building style guides, technical docs and complex ebooks as well as static sites and blogs.
 
 Installation
 ------------
@@ -37,7 +31,7 @@ Status
 
 Yarrow is an extraction from several existing private documentation projects. This repo is in alpha state, which means that many of the useful features are not yet folded into this codebase.
 
-Yarrow is being slowly developed as a part-time project to scratch a few itches. New features and bugfixes are pushed straight to master, and releases of the Gem are kept more or less in sync with the planned roadmap.
+Yarrow is being slowly developed as a part-time project to scratch a few itches. New features and bugfixes are pushed straight to `main`, and releases of the Gem are kept more or less in sync with the planned roadmap.
 
 Roadmap
 -------
@@ -46,13 +40,13 @@ A rough sketch of the project direction.
 
 | Version | Features |
 |---------|----------|
-| `0.3`   | **[Local dev server and asset pipeline](https://github.com/maetl/yarrow/issues/48)** |
-| `0.4`   | Default media type mapping, collector, markup converters |
-| `0.5`   | Content model/object mapping, template/site context |
-| `0.6`   | HTML tag helpers, default layout templates |
-| `0.7`   | Rake integration, task library |
-| `0.8`   | Generic command line runner |
-| `0.9`   | Refactoring, performance fixes, lock down API |
+| `0.6`   | Default media type mapping, graph collectors, markup converters |
+| `0.7`   | Content model/object mapping, template/site context |
+| `0.8`   | HTML publishing workflow |
+| `0.9`   | PDF publishing workflow |
+| `0.10`   | Media and video publishing workflow |
+| `0.11`   | Generic command line runner |
+| `1.0`   | Refactoring, performance fixes, lock down API |
 
 License
 -------

data/Rakefile

@@ -1,4 +1,18 @@
+$LOAD_PATH.unshift File.expand_path("./lib", __dir__)
+require "yarrow"
+
+task :version do
+  puts Yarrow::VERSION
+end
+
+task :build do
+  sh "gem build yarrow.gemspec"
+end
+
+task :publish do
+  sh "gem push yarrow-#{Yarrow::VERSION}.gem"
+end
 
 task :clean do
-  sh 'rm yarrow-*.gem'
+  sh "rm yarrow-*.gem"
 end

data/lib/yarrow.rb

@@ -1,25 +1,36 @@
-require 'hashie'
+require 'pathname'
 require 'yaml'
 
 require 'yarrow/version'
+require 'yarrow/extensions'
+require 'yarrow/symbols'
 require 'yarrow/logging'
+require 'yarrow/schema'
+require 'yarrow/config'
 require 'yarrow/configuration'
 require 'yarrow/console_runner'
 require 'yarrow/generator'
-require 'yarrow/assets'
+require 'yarrow/tools/front_matter'
+require 'yarrow/tools/content_utils'
 require 'yarrow/content/graph'
-#require 'yarrow/content/resource_expander'
+require 'yarrow/content/content_type'
+require 'yarrow/content/source'
 require 'yarrow/content/source_collector'
-require 'yarrow/html/asset_tags'
+require 'yarrow/content/collection_expander'
 require 'yarrow/output/mapper'
 require 'yarrow/output/generator'
 require 'yarrow/output/context'
+require 'yarrow/output/web/indexed_file'
 require 'yarrow/content_map'
-require 'yarrow/html'
-require 'yarrow/tools/front_matter'
 require 'yarrow/server'
 require 'yarrow/server/livereload'
 
+require 'yarrow/process/workflow'
+require 'yarrow/process/step_processor'
+require 'yarrow/process/expand_content'
+require 'yarrow/process/extract_source'
+require 'yarrow/process/project_manifest'
+
 # Dir[File.dirname(__FILE__) + '/yarrow/generators/*.rb'].each do |generator|
 #   require generator
 # end

data/lib/yarrow/config.rb

@@ -0,0 +1,59 @@
+# Replacement for the legacy Hashie::Mash/Module mixin configuration
+# pattern. This provides the same API (chaining nested attribute calls) but
+# handles schema validation and doesn’t pollute other namespaces.
+module Yarrow
+  module Config
+    # Basic defaults which can be used to populate HTML metadata as well as
+    # used in CLI listings and other generated docs. The default H1 if no
+    # specific output config or template themes are provided.
+    #
+    # For larger web publishing projects, this should be moved out into a
+    # template context or language/translation files to make it editable
+    # for a larger group of people.
+    Meta = Yarrow::Schema::Value.new(
+      :title,
+      :author
+      # :copyright TODO: what other basic details to include?
+    )
+
+    # Dev server config. This is mainly useful if you want to set up a specific
+    # chain of Rack middleware and handlers. If you don’t care about default
+    # directory indexes or port handling, you can completely ignore this.
+    #
+    # There are many better live reloading options available in JS, so the Rack
+    # infrastructure here should be ignored for UI-heavy jobs. It’s otherwise
+    # fine for slower-paced general purpose web publishing.
+    #
+    # The default index config could possibly move into a dedicated namespace
+    # in future if it makes sense to use the underlying graph infrastructure
+    # as a live lookup rather than a compiler-generated artifact. This would
+    # mean rather than doing a ls on the directory, the index pages would
+    # grab a list of entries out of a graph projection for the directory.
+    Server = Yarrow::Schema::Value.new(
+      :live_reload,
+      :auto_index,
+      :default_index,
+      :default_type,
+      :port,
+      :host,
+      :handler,
+      :middleware,
+      :root_dir
+    )
+
+    # Top level root config namespace. Source, content and output are directory
+    # paths and should be the only required defaults for a complete batch run.
+    #
+    # Additional server config is optional and only needed if running the dev
+    # server locally.
+    #
+    # TODO: meta should be union of Type::Optional and Config::Meta
+    Instance = Yarrow::Schema::Value.new(
+      source: Pathname,
+      content: Pathname,
+      output_dir: Pathname,
+      meta: Yarrow::Schema::Type::Any,
+      server: Yarrow::Schema::Type::Any
+    )
+  end
+end

data/lib/yarrow/configuration.rb

@@ -1,50 +1,6 @@
 module Yarrow
-  ##
-  # Hash-like object containing runtime configuration values. Can be accessed indifferently
-  # via object style lookups or symbol keys.
-  #
-  class Configuration < Hashie::Mash
+  class Configuration
     class << self
-      ##
-      # Provides access to the registered global configuration.
-      #
-      # If no configuration is registered, returns a blank object.
-      #
-      # @return [Yarrow::Configuration]
-      #
-      def instance
-        @@configuration ||= self.new
-      end
-
-      ##
-      # Loads and registers a global configuration instance.
-      #
-      # This will reset any previously initialized configuration.
-      #
-      # @param [String] path to YAML file
-      #
-      def register(file)
-        @@configuration = load(file)
-      end
-
-      ##
-      # Loads and registers a global configuration instance with
-      # library-provided defaults.
-      #
-      # This will reset any previously initialized configuration.
-      #
-      def register_defaults
-        @@configuration = load_defaults
-      end
-
-      ##
-      # Clears the global configuration to the empty default.
-      #
-      def clear
-        @@configuration = self.new
-      end
-
-      ##
       # Merges the given configuration or hash-like object with the
       # registered global configuration.
       #
@@ -54,18 +10,16 @@ module Yarrow
         instance.deep_merge!(config)
       end
 
-      ##
       # Loads a configuration object from the given YAML file.
       #
       # @param [String] path to YAML file
       #
-      # @return [Yarrow::Configuration]
+      # @return [Yarrow::Config]
       #
       def load(file)
-        self.new(YAML.load_file(file))
+        coerce_config_struct(YAML.load(File.read(file), symbolize_names: true))
       end
 
-      ##
       # Yarrow is distributed with a `defaults.yml` which provides minimum
       # boostrap configuration and default settings for various internal
       # classes. Use this method to automatically load these defaults.
@@ -74,21 +28,39 @@ module Yarrow
       def load_defaults
         load(File.join(File.dirname(__FILE__), 'defaults.yml'))
       end
-    end
-  end
 
-  ##
-  # Embeds global configuration access in a client object.
-  #
-  module Configurable
-    ##
-    # Provides access to the registered global configuration. This can
-    # be overriden by subclasses to customize behaviour (eg: in test cases).
-    #
-    # @return [Yarrow::Configuration]
-    #
-    def config
-      Configuration.instance
+      private
+
+      # TODO: this should be folded into the schema machinery with type coercions
+      def coerce_config_struct(config)
+        meta_obj = if config.key?(:meta)
+          Yarrow::Config::Meta.new(
+            title: config[:meta][:title],
+            author: config[:meta][:author]
+          )
+        else
+          nil
+        end
+
+        server_obj = if config.key?(:server)
+          Yarrow::Config::Server.new(**config[:server])
+        else
+          nil
+        end
+
+        # TODO: messy hack to get rid of Hashie::Mash, this should either be
+        # automated as part of the schema types or a default value should be
+        # generated here (eg: `"#{Dir.pwd}/docs"`)
+        out_dir_or_string = config[:output_dir] || ""
+
+        Yarrow::Config::Instance.new(
+          output_dir: Pathname.new(File.expand_path(out_dir_or_string)),
+          source: Pathname.new(File.expand_path(out_dir_or_string)),
+          content: Pathname.new(File.expand_path(out_dir_or_string)),
+          meta: meta_obj,
+          server: server_obj
+        )
+      end
     end
   end
 

data/lib/yarrow/content/collection_expander.rb

@@ -0,0 +1,218 @@
+module Yarrow
+  module Content
+    class CollectionExpander
+      include Yarrow::Tools::FrontMatter
+
+      def initialize(content_types=nil)
+        @content_types = content_types || [
+          Yarrow::Content::ContentType.from_name(:pages)
+        ]
+      end
+
+      def expand(graph)
+        @content_types.each do |content_type|
+          expand_nested(graph, content_type)
+        end
+      end
+
+      def expand_nested(graph, content_type)
+        type = content_type.collection
+        exts = content_type.extensions
+
+        # If match path represents entire content dir, then include the entire
+        # content dir instead of scanning from a subfolder matching the name of
+        # the collection.
+        start_node = if content_type.match_path == "."
+          graph.n(:root)
+        else
+          graph.n(:root).out(name: type.to_s)
+        end
+
+        # Extract metadata from given start node
+        data = extract_metadata(start_node, type)
+
+        # Collects all nested collections in the subgraph for this content type
+        subcollections = {}
+        index = nil
+
+        # Define alias for accessing metadata in the loop
+        metadata = data
+
+        # Scan and collect all nested directories under the top level source
+        start_node.depth_first.each do |node|
+          if node.label == :directory
+            # Check if this entry has metadata defined at the top level
+            if data[:collections]
+              item = data[:collections].find { |c| c[:slug] == node.props[:slug] }
+              metadata = item if item
+            end
+
+            # Create a collection node representing a collection of documents
+            index = graph.create_node do |collection_node|
+              collection_node.label = :collection
+              collection_node.props[:type] = type
+              collection_node.props[:name] = node.props[:name]
+              collection_node.props[:slug] = node.props[:slug]
+              collection_node.props[:title] = metadata[:title]
+
+              # Override default status so that mapped index collections always show
+              # up in the resulting document manifest, when they don’t have
+              # associated metadata. This is the opposite of how individual pieces
+              # of content behave (default to draft status if one isn’t supplied).
+              collection_node.props[:status] = if data[:status]
+                data[:status]
+              else
+                "published"
+              end
+
+              # TODO: URL generation might need to happen elsewhere
+              collection_node.props[:url] = if data[:url]
+                data[:url]
+              else
+                "#{node.props[:path].split('./content').last}/"
+              end
+            end
+
+            # Add this collection id to the lookup table for edge construction
+            subcollections[node.props[:path]] = index
+
+            # Join the collection to its parent
+            unless node.props[:slug] == type.to_s || !subcollections.key?(node.props[:entry].parent.to_s)
+              graph.create_edge do |edge|
+                edge.label = :child
+                edge.from = subcollections[node.props[:entry].parent.to_s].id
+                edge.to = index.id
+              end
+            end
+          end
+        end
+
+        subcollections.each do |path, index|
+          # Group files matching the same slug under a common key
+          objects = graph.n(path: path).out(:file).all.select do |file|
+            file.props[:name].end_with?(*exts)
+          end.group_by do |file|
+            file.props[:slug]
+          end
+
+          build_content_nodes(graph, objects, type, index)
+        end
+      end
+
+      # Extract collection level configuration/metadata from the root node for
+      # this content type.
+      def extract_metadata(node, type)
+        # TODO: support _index or _slug convention as well
+        meta_file = node.out(slug: type.to_s).first
+
+        if meta_file
+          # Process metadata and add it to the collection node
+          # TODO: pass in content converter object
+          # TODO: index/body content by default if extracted from frontmatter
+          body, data = process_content(meta_file.props[:entry])
+        else
+          # Otherwise, assume default collection behaviour
+          data = {}
+        end
+
+        # Generate a default title if not provided in metadata
+        unless data.key?(:title)
+          data[:title] = type.to_s.capitalize
+        end
+
+        data
+      end
+
+      def build_content_nodes(graph, objects, type, parent_index)
+        # TODO: this may need to use a strategy that can be overriden
+        content_type = Yarrow::Symbols.to_singular(type)
+
+        # Process collected content objects and generate entity nodes
+        objects.each do |name, sources|
+          item_node = graph.create_node do |node|
+            # TODO: Rename this to :entry and support similar fields to Atom
+            node.label = :item
+            node.props[:name] = name
+            node.props[:type] = content_type
+
+            meta = {}
+            content = ""
+
+            sources.each do |source|
+              body, data = process_content(source.props[:entry])
+              meta.merge!(data) unless data.nil?
+              content << body unless body.nil?
+            end
+
+            if meta[:url]
+              # If a URL is explicitly proided in metadata then use it
+              node.props[:url] = meta[:url]
+            elsif meta[:permalink]
+              # Support for legacy permalink attribute
+              node.props[:url] = meta[:permalink]
+            else
+              # Default URL generation strategy when no explicit URL is provided
+              # TODO: collection nodes need URL generation too
+              # TODO: replace this with URL generation strategy
+              # TODO: slug vs name - why do some nodes have 2 and some 3 props?
+              node.props[:url] = if parent_index.props[:name].to_sym == parent_index.props[:type]
+                "/#{parent_index.props[:type]}/#{name}"
+              else
+                "/#{parent_index.props[:type]}/#{parent_index.props[:slug]}/#{name}"
+              end
+            end
+
+            # For now, we are storing title, url, etc on the top-level item.
+            node.props[:title] = meta[:title]
+
+            # TODO: What belongs on the entity and what belongs on the item?
+            entity_props = meta.merge(body: content, name: meta[:id], url: node.props[:url])
+
+
+            # TODO: consider whether to provide `body` on the item/document or at
+            # the custom content type level.
+            begin
+              content_struct = Yarrow::Symbols.to_const(content_type)
+            rescue
+              # No immutable struct found: fall back to slower dynamically typed open struct
+              require "ostruct"
+              content_struct = OpenStruct
+            end
+
+            node.props[:entity] = content_struct.new(entity_props)
+          end
+
+          # Connect entity with source content
+          sources.each do |source|
+            graph.create_edge do |edge|
+              edge.label = :source
+              edge.from = item_node
+              edge.to = source.id
+            end
+          end
+
+          # Connect entity with parent collection
+          graph.create_edge do |edge|
+            edge.label = :child
+            edge.from = parent_index
+            edge.to = item_node
+          end
+        end
+      end
+
+      # Workaround for handling meta and content source in multiple files or a single
+      # file with front matter.
+      def process_content(path)
+        case path.extname
+        when '.htm', '.md'
+          read_split_content(path.to_s, symbolize_keys: true)
+        # when '.md'
+        #   body, data = read_split_content(path.to_s, symbolize_keys: true)
+        #   [Kramdown::Document.new(body).to_html, data]
+        when '.yml'
+          [nil, YAML.load(File.read(path.to_s), symbolize_names: true)]
+        end
+      end
+    end
+  end
+end