glim 0.1.1

data/lib/cache.rb

@@ -0,0 +1,66 @@
+require 'fileutils'
+
+module Glim
+  class Cache
+    CACHE_PATH = '.cache/glim/data.bin'
+
+    class << self
+      def load
+        cache
+      end
+
+      def save
+        unless @cache.nil?
+          FileUtils.mkdir_p(File.dirname(CACHE_PATH))
+          open(CACHE_PATH, 'w') do |io|
+            Marshal.dump(cache, io)
+          end
+        end
+      end
+
+      def track_updates=(flag)
+        @updates = flag ? {} : nil
+      end
+
+      def updates
+        @updates
+      end
+
+      def merge!(updates)
+        updates.each do |group, paths|
+          (cache[group] ||= {}).merge!(paths)
+        end
+      end
+
+      def getset(path, group = :default)
+        begin
+          mtime = File.stat(path).mtime
+          if record = cache.dig(group, path)
+            if mtime == record['modified']
+              return record['data']
+            end
+          end
+
+          record = {
+            'modified' => mtime,
+            'data'     => yield,
+          }
+
+          (cache[group] ||= {})[path] = record
+          (@updates[group] ||= {})[path] = record if @updates
+
+          record['data']
+        rescue Errno::ENOENT
+          $log.warn("File does not exist: #{path}")
+          nil
+        end
+      end
+
+      private
+
+      def cache
+        @cache ||= open(CACHE_PATH) { |io| Marshal.load(io) } rescue {}
+      end
+    end
+  end
+end

data/lib/commands.rb

@@ -0,0 +1,261 @@
+module Glim
+  module Commands
+    def self.build(config)
+      output_dir = File.expand_path(config['destination'])
+      files      = config.site.files_and_documents.select { |file| file.write? }
+      symlinks   = (config.site.symlinks || []).map { |link| [ File.expand_path(File.join(link[:data]['domain'] || '.', link[:name]), output_dir), link[:realpath] ] }.to_h
+
+      output_paths = files.map { |file| file.output_path(output_dir) }
+      output_paths.concat(symlinks.keys)
+
+      delete_files, delete_dirs = items_in_directory(output_dir, skip: config['keep_files'])
+      deleted = delete_items(delete_files, delete_dirs, keep: output_paths)
+      created, updated, warnings, errors = *generate(output_dir, config['jobs'] || 7, files)
+
+      symlinks.each do |dest, path|
+        FileUtils.mkdir_p(File.dirname(dest))
+        begin
+          File.symlink(path, dest)
+          created << dest
+        rescue Errno::EEXIST
+          if File.readlink(dest) != path
+            File.unlink(dest)
+            File.symlink(path, dest)
+            updated << dest
+          end
+        end
+      end
+
+      [ [ 'Created', created ], [ 'Deleted', deleted ], [ 'Updated', updated ] ].each do |label, files|
+        unless files.empty?
+          STDERR.puts "==> #{label} #{files.count} #{files.count == 1 ? 'File' : 'Files'}"
+          STDERR.puts files.map { |path| Util.relative_path(path, output_dir) }.sort.join(', ')
+        end
+      end
+
+      unless warnings.empty?
+        STDERR.puts "==> #{warnings.count} #{warnings.count == 1 ? 'Warnings' : 'Warning'}"
+        warnings.each do |message|
+          STDERR.puts message
+        end
+      end
+
+      unless errors.empty?
+        STDERR.puts "==> Stopped After #{errors.count} #{errors.count == 1 ? 'Error' : 'Errors'}"
+        errors.each do |arr|
+          arr.each_with_index do |err, i|
+            STDERR.puts err.gsub(/^/, '  '*i)
+          end
+        end
+      end
+    end
+
+    def self.clean(config)
+      files, dirs = items_in_directory(File.expand_path(config['destination']), skip: config['keep_files'])
+
+      if config['dry_run']
+        if files.empty?
+          STDOUT.puts "No files to delete"
+        else
+          files.each do |file|
+            STDOUT.puts "Delete #{Util.relative_path(file, File.expand_path(config['source']))}"
+          end
+        end
+      else
+        deleted = delete_items(files, dirs)
+        STDOUT.puts "Deleted #{deleted.count} #{deleted.count == 1 ? 'File' : 'Files'}."
+      end
+    end
+
+    def self.profile(config)
+      Profiler.enabled = true
+
+      site = Profiler.run("Setting up site") do
+        config.site
+      end
+
+      Profiler.run("Loading cache") do
+        Glim::Cache.load
+      end
+
+      files = []
+
+      Profiler.run("Loading pages") do
+        files.concat(site.files)
+      end
+
+      Profiler.run("Loading collections") do
+        files.concat(site.documents)
+      end
+
+      Profiler.run("Generating virtual pages") do
+        files.concat(site.generated_files)
+      end
+
+      files = files.select { |file| file.frontmatter? }
+
+      Profiler.run("Expanding liquid tags") do
+        files.each { |file| file.content('post-liquid') }
+      end
+
+      Profiler.run("Transforming pages") do
+        files.each { |file| file.content('pre-output') }
+      end
+
+      Profiler.run("Creating final output (layout)") do
+        files.each { |file| file.output }
+      end
+
+      Profiler.enabled = false
+    end
+
+    # ===========
+    # = Private =
+    # ===========
+
+    def self.items_in_directory(dir, skip: [])
+      files, dirs = [], []
+
+      begin
+        Find.find(dir) do |path|
+          next if path == dir
+          Find.prune if skip.include?(File.basename(path))
+
+          if File.file?(path) || File.symlink?(path)
+            files << path
+          elsif File.directory?(path)
+            dirs << path
+          else
+            $log.warn("Unknown entry: #{path}")
+          end
+        end
+      rescue Errno::ENOENT
+      end
+
+      [ files, dirs ]
+    end
+
+    def self.delete_items(files, dirs, keep: [])
+      res = []
+
+      keep_files = Set.new(keep)
+      files.each do |path|
+        unless keep_files.include?(path)
+          begin
+            File.unlink(path)
+            res << path
+          rescue => e
+            $log.error("Error unlinking ‘#{path}’: #{e}\n")
+          end
+        end
+      end
+
+      dirs.sort.reverse.each do |path|
+        begin
+          Dir.rmdir(path)
+        rescue Errno::ENOTEMPTY => e
+          # Ignore
+        rescue => e
+          $log.error("Error removing directory ‘#{path}’: #{e}\n")
+        end
+      end
+
+      res
+    end
+
+    def self.generate(output_dir, number_of_jobs, files)
+      Profiler.run("Creating pages") do
+        if number_of_jobs == 1
+          generate_subset(output_dir, files)
+        else
+          generate_async(output_dir, files.shuffle, number_of_jobs)
+        end
+      end
+    end
+
+    def self.generate_async(output_dir, files, number_of_jobs)
+      total  = files.size
+      slices = number_of_jobs.times.map do |i|
+        first = (total *    i  / number_of_jobs).ceil
+        last  = (total * (i+1) / number_of_jobs).ceil
+        files.shift(last-first)
+      end
+
+      Glim::Cache.track_updates = true
+      semaphore = Mutex.new
+      created, updated, warnings, errors = [], [], [], []
+
+      threads = slices.each_with_index.map do |files_slice, i|
+        pipe_rd, pipe_wr = IO.pipe
+        pid = fork do
+          start = Time.now
+          pipe_rd.close
+          created, updated, warnings, errors = *generate_subset(output_dir, files_slice)
+          pipe_wr << Marshal.dump({
+            'cache_updates' => Glim::Cache.updates,
+            'created'       => created,
+            'updated'       => updated,
+            'warnings'      => warnings,
+            'errors'        => errors,
+            'duration'      => Time.now - start,
+            'id'            => i,
+          })
+          pipe_wr.close
+        end
+
+        Process.detach(pid)
+
+        Thread.new do
+          pipe_wr.close
+          res = Marshal.load(pipe_rd)
+          semaphore.synchronize do
+            Glim::Cache.merge!(res['cache_updates'])
+            created  += res['created']
+            updated  += res['updated']
+            warnings += res['warnings']
+            errors   += res['errors']
+            $log.debug("Wrote #{files_slice.size} pages in #{res['duration']} seconds (thread #{res['id']})") if Profiler.enabled
+          end
+        end
+      end
+
+      threads.each { |thread| thread.join }
+
+      [ created, updated, warnings, errors ]
+    end
+
+    def self.generate_subset(output_dir, files)
+      created, updated, warnings, errors = [], [], [], []
+
+      for file in files do
+        dest = file.output_path(output_dir)
+        file_exists = File.exists?(dest)
+
+        FileUtils.mkdir_p(File.dirname(dest))
+        if file.frontmatter?
+          begin
+            if !file_exists || File.read(dest) != file.output
+              (file_exists ? updated : created) << dest
+              File.unlink(dest) if file_exists
+              File.write(dest, file.output)
+            end
+            warnings.concat(file.warnings.map { |warning| "#{file}: #{warning}" }) unless file.warnings.nil?
+          rescue Glim::Error => e
+            errors << [ "Unable to create output for: #{file}", *e.messages ]
+            break
+          rescue => e
+            errors << [ "Unable to create output for: #{file}", e.to_s, e.backtrace.join("\n") ]
+            break
+          end
+        else
+          unless File.file?(dest) && File.file?(file.path) && File.stat(dest).ino == File.stat(file.path).ino
+            File.unlink(dest) if file_exists
+            File.link(file.path, dest)
+          end
+        end
+      end
+
+      [ created, updated, warnings, errors ]
+    end
+  end
+end

data/lib/exception.rb

@@ -0,0 +1,18 @@
+module Glim
+  class Error < ::RuntimeError
+    attr_reader :message, :previous
+
+    def initialize(message, previous = nil)
+      @message, @previous = message, previous
+    end
+
+    def messages
+      res = [ @message ]
+      e = self
+      while e.respond_to?(:previous) && (e = e.previous)
+        res << e.message
+      end
+      res
+    end
+  end
+end

data/lib/liquid_ext.rb

@@ -0,0 +1,249 @@
+require 'kramdown'
+require 'liquid'
+
+module Glim
+  module LiquidFilters
+    def markdownify(input)
+      Profiler.group('markdownify') do
+        if defined?(MultiMarkdown)
+          MultiMarkdown.new("\n" + input, 'snippet', 'no_metadata').to_html
+        else
+          options  = @context['site']['kramdown'].map { |key, value| [ key.to_sym, value ] }.to_h
+          document = Kramdown::Document.new(input, options)
+          @context['warnings'].concat(document.warnings) if options[:show_warnings] && @context['warnings']
+          document.to_html
+        end unless input.nil?
+      end
+    end
+
+    def slugify(input)
+      Util.slugify(input) unless input.nil?
+    end
+
+    def xml_escape(input)
+      input.encode(:xml => :attr).gsub(/\A"|"\z/, '') unless input.nil?
+    end
+
+    def cgi_escape(input)
+      CGI.escape(input) unless input.nil?
+    end
+
+    def absolute_url(path)
+      unless path.nil?
+        site, page = URI(@context['site']['url']), URI(@context['page']['url'])
+        host, port = @context['site']['host'], @context['site']['port']
+        if page.relative? || (site.host == host && site.port == port)
+          site.merge(URI(path)).to_s
+        else
+          page.merge(URI(path)).to_s
+        end
+      end
+    end
+
+    def relative_url(other)
+      helper = lambda do |base, other|
+        base_url, other_url = URI(base), URI(other)
+        if other_url.absolute? && base_url.host == other_url.host
+          other_url.path
+        else
+          other
+        end
+      end
+
+      unless other.nil?
+        site, page = URI(@context['site']['url']), URI(@context['page']['url'])
+        host, port = @context['site']['host'], @context['site']['port']
+        if page.relative? || (site.host == host && site.port == port)
+          helper.call(@context['site']['url'], other)
+        else
+          helper.call(@context['page']['url'], other)
+        end
+      end
+    end
+
+    def path_to_url(input)
+      if file = Jekyll.sites.last.links[input]
+        file.url
+      else
+        raise Glim::Error.new("path_to_url: No file found for: #{input}")
+      end unless input.nil?
+    end
+
+    def date_to_xmlschema(input)
+      Liquid::Utils.to_date(input).localtime.xmlschema unless input.nil?
+    end
+
+    def date_to_rfc822(input)
+      Liquid::Utils.to_date(input).localtime.rfc822 unless input.nil?
+    end
+
+    def date_to_string(input)
+      Liquid::Utils.to_date(input).localtime.strftime("%d %b %Y") unless input.nil?
+    end
+
+    def date_to_long_string(input)
+      Liquid::Utils.to_date(input).localtime.strftime("%d %B %Y") unless input.nil?
+    end
+
+    def where(input, property, value)
+      if input.respond_to?(:select) && property && value
+        input = input.values if input.is_a?(Hash)
+        input.select { |item| get_property(item, property) == value }
+      else
+        input
+      end
+    end
+
+    def group_by(input, property)
+      if input.respond_to?(:group_by) && property
+        groups = input.group_by { |item| get_property(item, property) }
+        groups.map { |key, value| { "name" => key, "items" => value, "size" => value.size } }
+      else
+        input
+      end
+    end
+
+    def group_by_exp(input, variable, expression)
+      if input.respond_to?(:group_by)
+        parsed_expr = Liquid::Variable.new(expression, Liquid::ParseContext.new)
+        @context.stack do
+          groups = input.group_by do |item|
+            @context[variable] = item
+            parsed_expr.render(@context)
+          end
+          groups.map { |key, value| { "name" => key, "items" => value, "size" => value.size } }
+        end
+      else
+        input
+      end
+    end
+
+    private
+
+    def get_property(obj, property)
+      if obj.respond_to?(:to_liquid)
+        property.to_s.split('.').reduce(obj.to_liquid) do |mem, key|
+          mem[key]
+        end
+      elsif obj.respond_to?(:data)
+        obj.data[property.to_s]
+      else
+        obj[property.to_s]
+      end
+    end
+  end
+
+  module LiquidTags
+    class PostURL < Liquid::Tag
+      def initialize(tag_name, markup, options)
+        super
+        @post_name = markup.strip
+      end
+
+      def render(context)
+        if file = Jekyll.sites.last.post_links[@post_name]
+          file.url
+        else
+          raise Glim::Error.new("post_url: No post found for: #{@post_name}")
+        end
+      end
+    end
+
+    class Link < Liquid::Tag
+      def initialize(tag_name, markup, options)
+        super
+        @relative_path = markup.strip
+      end
+
+      def render(context)
+        if file = Jekyll.sites.last.links[@relative_path]
+          file.url
+        else
+          raise Glim::Error.new("link: No file found for: #{@relative_path}")
+        end
+      end
+    end
+
+    class HighlightBlock < Liquid::Block
+      def initialize(tag_name, markup, tokens)
+        super
+
+        if markup =~ /^([a-zA-Z0-9.+#_-]+)((\s+\w+(=(\w+|"[^"]*"))?)*)\s*$/
+          @language, @options = $1, $2.scan(/(\w+)(?:=(?:(\w+)|"([^"]*)"))?/).map do |key, value, list|
+            [ key.to_sym, list ? list.split : (value || true) ]
+          end.to_h
+        else
+          @language, @options = nil, {}
+          $log.error("Unable to parse highlight tag: #{markup}") unless markup.strip.empty?
+        end
+
+        begin
+          require 'rouge'
+        rescue LoadError => e
+          $log.warn("Unable to load the rouge gem required by the highlight tag: #{e}")
+        end
+      end
+
+      def render(context)
+        source = super.to_s.gsub(/\A[\r\n]+|[\r\n]+\z/, '')
+
+        if defined?(Rouge)
+          rouge_options = {
+            :line_numbers => @options[:linenos] == true ? 'inline' : @options[:linenos],
+            :wrap         => false,
+            :css_class    => 'highlight',
+            :gutter_class => 'gutter',
+            :code_class   => 'code'
+          }.merge(@options)
+
+          lexer     = Rouge::Lexer.find_fancy(@language, source) || Rouge::Lexers::PlainText
+          formatter = Rouge::Formatters::HTMLLegacy.new(rouge_options)
+          source    = formatter.format(lexer.lex(source))
+
+          $log.warn("No language specified in highlight tag. Will use #{lexer.class.name} to parse the code.") if @language.nil?
+        end
+
+        code_attributes = @language ? " class=\"language-#{@language.tr('+', '-')}\" data-lang=\"#{@language}\"" : ""
+        "<figure class=\"highlight\"><pre><code#{code_attributes}>#{source.chomp}</code></pre></figure>"
+      end
+    end
+  end
+
+  def self.preprocess_template(source)
+    source = source.gsub(/({%-? include )([\w.\/-]+)(.*?)(-?%})/) do
+      prefix, include, variables, suffix = $1, $2, $3, $4
+      unless variables.strip.empty?
+        variables = ', ' + variables.scan(/(\w+)=(.*?)(?=\s)/).map { |key, value| "include_#{key}: #{value}" }.join(', ') + ' '
+      end
+
+      "#{prefix}\"#{include}\"#{variables}#{suffix}"
+    end
+
+    source.gsub!(/({{-? include)\.(.*?}})/) { "#$1_#$2" }
+    source.gsub!(/({%-? .+? include)\.(.*?%})/) { "#$1_#$2" }
+
+    source
+  end
+
+  class LocalFileSystem
+    def initialize(*paths)
+      @paths = paths.reject { |path| path.nil? }
+    end
+
+    def read_template_file(name)
+      @cache ||= {}
+      unless @cache[name]
+        paths = @paths.map { |path| File.join(path, name) }
+        if file = paths.find { |path| File.exist?(path) }
+          @cache[name] = Glim.preprocess_template(File.read(file))
+        end
+      end
+      @cache[name]
+    end
+  end
+end
+
+Liquid::Template.register_filter(Glim::LiquidFilters)
+Liquid::Template.register_tag('post_url', Glim::LiquidTags::PostURL)
+Liquid::Template.register_tag('link', Glim::LiquidTags::Link)
+Liquid::Template.register_tag("highlight", Glim::LiquidTags::HighlightBlock)