raindown 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0250c356fb715c70bb5620ebb20f3d9f92775398b1b002c71eec906f4cf3ad20
-  data.tar.gz: a601014e5ce54762c17db6d888067cd66239a64169ab453621036d95ca82a177
+  metadata.gz: ab2295aa7f9468fa67aaf9d8192de859220c37cec922ef558607730a0544a86a
+  data.tar.gz: 0ede0a653f452c0a9f12f9a73c35dce3df85a12e9ff35350d741ae9c7c0e3522
 SHA512:
-  metadata.gz: 975fbcedd3fc3a2e3c2b853173ac5f5eb882db921462fc4dba2d15cc35e7e8278ff0c44a89e986f8c967dcc73c1f80d59f1348850cddb06e92d409484d18f005
-  data.tar.gz: 2042a62352c54e59801d300cd49fa5ea7d7ffc65a8f01bd1aa0f213151606d947a7ea614239e148d36e5ccff1f0340714870dc1a383f695b7266990e58632c57
+  metadata.gz: f72d0180ff9b4b310c25efa2872e59dc5d31da9dacff67d7d047737cab25e3b6647b2d74fb2a130c9354f0c434cf3a57e5a7c0750a54d762a47121901abcbb83
+  data.tar.gz: 3bcc85d7eb3b2055f374028f2bdcc3e561530ea36739e655d96c0b26f5038d7e83104b7201ee9b2654ecd8ea6e550c96d9733412bb2ff07a7f31f5858630c003

data/bin/raindown

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/raindown/cli'
+
+Raindown::CLI.run(ARGV)

data/lib/elements.rb

@@ -2,19 +2,19 @@
 
 require 'plugs'
 
-module Rain
+module Raindown
   class Elements
     include Plugs
 
-    plug(:toc) do
+    plug(:endpoint) do
       plug(:lexeme) do
-        require_relative 'lexemes/toc_lexeme'
-        TOCLexeme
+        require_relative 'lexemes/endpoint_lexeme'
+        EndpointLexeme
       end
 
       plug(:node) do
-        require_relative 'nodes/toc_node'
-        TOCNode
+        require_relative 'nodes/endpoint_node'
+        EndpointNode
       end
     end
 

data/lib/lexemes/endpoint_lexeme.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'antlers/api'
+
+module Raindown
+  module EndpointLexeme
+    include Antlers::Lexeme
+
+    module_function
+
+    KEYWORDS = ['endpoint:', ':endpoint'].freeze
+
+    def match?(keywords:, **)
+      KEYWORDS.include?(keywords.keys.first)
+    end
+
+    def lexeme(keywords:, **)
+      return { endpoint_def: keywords } if keywords.key?('endpoint:')
+
+      { endpoint_end: 'level_1' }
+    end
+  end
+end

data/lib/lexemes/list_lexeme.rb

@@ -2,7 +2,7 @@
 
 require 'antlers/api'
 
-module Rain
+module Raindown
   module ListLexeme
     include Antlers::Lexeme
 

data/lib/nodes/endpoint_node.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'antlers/api'
+require_relative '../raindown/directives'
+
+module Raindown
+  # Holds an endpoint's curl snippet and response, which render as the code blocks they are,
+  # between Directives markers for its directive and end tag.
+  class EndpointNode < Antlers::BranchNode
+    DEF_KEY = :endpoint_def
+    END_KEY = :endpoint_end
+
+    def initialize(name:, directive: 'endpoint:', **)
+      super(name:, **)
+
+      @directive = directive
+    end
+
+    def render(**)
+      "#{Directives.marker(@directive)}\n#{super.sub(/\A\n+/, '')}#{Directives.marker(':endpoint')}"
+    end
+
+    class << self
+      def match?(segment:)
+        segment[DEF_KEY]
+      end
+
+      def build(segment:, **)
+        new(name: :endpoint, directive: segment[DEF_KEY].map { |keyword, value| [keyword, value].compact.join(' ') }.join(' '))
+      end
+    end
+  end
+end

data/lib/nodes/list_node.rb

@@ -2,7 +2,7 @@
 
 require 'antlers/api'
 
-module Rain
+module Raindown
   class ListNode < Antlers::BranchNode
     include Antlers::Props
     include Antlers::Variables

data/lib/raindown/bare_directives.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative 'code_blocks'
+require_relative 'front_matter'
+
+module Raindown
+  # Raindown files (.rd/.raindown) can write a directive like <{ if: draft }> as a bare if: draft on its own line.
+  # Markdown files opt in with the --bare flag. Nodes like toc: and files: are handled by NodeBlocks, before the rest.
+  #
+  # Bare directives have no end tag. A bare block directive closes itself after the content it applies to,
+  # so where a block needs to wrap more than that, write it with delimiters and an end tag instead:
+  #   <{ if: draft }>
+  #   ...
+  #   <{ :if }>
+  module BareDirectives
+    EXTENSIONS = %w[.rd .raindown].freeze
+    # The block directives, and the method that finds the line their end tag goes after.
+    BLOCK_KEYWORDS = { 'endpoint:' => :after_endpoint, 'for:' => :after_block, 'if:' => :after_block,
+                       'list:' => :after_block }.freeze
+    # Four spaces of indent would make the line a Markdown code block.
+    LINE = /\A( {0,3})((#{BLOCK_KEYWORDS.keys.join('|')})(?:[ \t]+\S.*?)?)[ \t]*$/
+    # An end tag on its own line, like one a bare block directive above added, ends a block.
+    END_TAG = /\A {0,3}<\{\s*:[a-z][\w-]*\s*\}>[ \t]*$/
+
+    module_function
+
+    def enabled?(path: nil, bare: false)
+      bare || file?(path)
+    end
+
+    def file?(path)
+      return false if path.nil?
+
+      EXTENSIONS.include?(File.extname(path).downcase)
+    end
+
+    # Wrap each bare directive in Antlers delimiters and close its block, leaving code and front matter as-is.
+    def expand(content)
+      lines = content.lines
+      index = FrontMatter.line_count(lines)
+
+      while index < lines.length
+        block = CodeBlocks.at(lines, index)
+        index = (block ? block[:close] : expand_line(lines, index)) + 1
+      end
+
+      lines.join
+    end
+
+    # The directive in Antlers delimiters, or nil when the line isn't a bare directive.
+    def wrap(line)
+      line.sub(LINE) { "#{::Regexp.last_match(1)}<{ #{::Regexp.last_match(2)} }>" } if line.match?(LINE)
+    end
+
+    # Returns the index of the last line it handled. That's the directive itself, so the lines it holds
+    # are expanded too, and a bare directive inside it closes before it does.
+    def expand_line(lines, index)
+      match = lines[index].match(LINE)
+      return index unless match
+
+      lines[index] = wrap(lines[index])
+      close_block(lines, index, indent: match[1], keyword: match[3])
+      index
+    end
+
+    def close_block(lines, index, indent:, keyword:)
+      last = public_send(BLOCK_KEYWORDS[keyword], lines, index + 1) || index
+      lines[last] = "#{lines[last]}\n" unless lines[last].end_with?("\n")
+      lines.insert(last + 1, "#{indent}<{ :#{keyword.delete_suffix(':')} }>\n")
+    end
+
+    # The last line of the curl snippet and response below an endpoint.
+    def after_endpoint(lines, index)
+      CodeBlocks.endpoint(lines, index).last&.fetch(:close)
+    end
+
+    # The last line of the Markdown block below, after any blank lines: up to the next blank line or end tag,
+    # with fenced code kept whole. Nil when there's nothing below.
+    def after_block(lines, index)
+      index = CodeBlocks.skip_blank_lines(lines, index)
+      last = nil
+
+      while index < lines.length && !lines[index].strip.empty? && !lines[index].match?(END_TAG)
+        last = CodeBlocks.at(lines, index)&.fetch(:close) || index
+        index = last + 1
+      end
+
+      last
+    end
+  end
+end

data/lib/raindown/cli.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'trees'
+require_relative '../raindown'
+require_relative 'rewriter'
+require_relative 'watcher'
+
+module Raindown
+  module CLI
+    extend Trees
+
+    USAGE = <<~TEXT
+      Usage: raindown FILE [options]
+
+      Renders FILE to HTML, then re-renders it every time it is saved.
+
+      Options:
+        -o, --output PATH   Write output to PATH instead of FILE.html beside FILE
+        --once              Render once without watching, printing to stdout unless -o is given
+        --stdin             Render Markdown from stdin once, without formatting FILE, which only needs
+                            its extension (to allow bare directives) and can be unsaved
+        --bare              Allow bare directives like :toc in Markdown files
+        -x, --execute       Run endpoint curl snippets that have no output yet
+        -h, --help          Show this message
+    TEXT
+
+    OPTIONS = { output: ['-o :output', '--output :output'], bare: ['--bare'], execute: ['-x', '--execute'],
+                once: ['--once'], stdin: ['--stdin'] }.freeze
+
+    # Trees matches tokens in order, so there's a line for every order and spelling of the options.
+    OPTION_LINES = (0..OPTIONS.size).flat_map { |count| OPTIONS.keys.permutation(count).to_a }.flat_map do |names|
+      [[]].product(*names.map { |name| OPTIONS[name] }).map { |flags| [names, flags.flatten] }
+    end
+
+    OPTION_LINES.each do |names, flags|
+      # Endpoints run while formatting, which --stdin skips.
+      next if names.include?(:stdin) && names.include?(:execute)
+
+      command = if names.include?(:stdin) then :render_stdin
+                elsif names.include?(:once) then :render
+                else :watch
+                end
+      options = { bare: names.include?(:bare), execute: names.include?(:execute) }
+      pattern = [':path', *flags].join(' ')
+
+      if names.include?(:output)
+        line(pattern) { |path, output| execute { send(command, path, output, **options) } }
+      else
+        line(pattern) { |path| execute { send(command, path, **options) } }
+      end
+    end
+
+    # The :path line would match a help flag as the file, so help is checked first.
+    def self.run(argv)
+      abort(USAGE) if argv.empty? || argv.intersect?(%w[-h --help])
+
+      super || abort("Unknown arguments: #{argv.join(' ')}\n\n#{USAGE}")
+    end
+
+    class << self
+      private
+
+      def render(path, output = nil, bare: false, execute: false)
+        ensure_exists(path)
+        Nodes.load(path)
+
+        metadata = Pages.for(path)
+        Rewriter.update(path:, bare:, execute:, metadata:)
+        write(output, Raindown.render_file(path:, bare:, metadata:))
+      end
+
+      def render_stdin(path, output = nil, bare: false, **)
+        Nodes.load(path)
+        write(output, Raindown.render(markdown: $stdin.read, metadata: Pages.for(path), bare: bare || BareDirectives.file?(path),
+                                      hide_directives: Config.for(path)[:hide_directives]))
+      end
+
+      def watch(path, output = nil, bare: false, execute: false)
+        ensure_exists(path)
+        Nodes.load(path)
+
+        Watcher.new(path:, output:, bare:, execute:).start
+      end
+
+      def write(output, html)
+        output ? File.write(output, html) : puts(html)
+      end
+
+      def ensure_exists(path)
+        abort("File not found: #{path}") unless File.exist?(path)
+      end
+    end
+  end
+end

data/lib/raindown/code.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'commonmarker'
+
+module Raindown
+  # Finds code in Markdown the way Commonmarker reads it: code spans and fenced, indented or quoted code blocks.
+  module Code
+    module_function
+
+    # Byte ranges of code in the Markdown, from Commonmarker's line and column positions.
+    def ranges(markdown)
+      line_offsets = markdown.each_line.reduce([0]) { |offsets, line| offsets << (offsets.last + line.bytesize) }
+      ranges = []
+
+      Commonmarker.parse(markdown, options: MARKDOWN_OPTIONS).walk do |node|
+        ranges << range(node.source_position, line_offsets) if %i[code code_block].include?(node.type)
+      end
+
+      ranges
+    end
+
+    def range(position, line_offsets)
+      start = line_offsets[position[:start_line] - 1] + position[:start_column] - 1
+      # A block that ends with its last line ends at column 0 of the line after.
+      finish = line_offsets[position[:end_line] - 1] + position[:end_column]
+      start...finish
+    end
+
+    # Swaps braces in code for the placeholders that render as braces, so Antlers leaves them alone.
+    def hide_braces(markdown)
+      ranges(markdown).reverse.each_with_object(markdown.b) do |range, bytes|
+        bytes[range] = bytes[range].force_encoding('UTF-8').gsub(/[{}]/, CODE_BRACES).b
+      end.force_encoding('UTF-8')
+    end
+  end
+end

data/lib/raindown/code_blocks.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Raindown
+  # Finds fenced code blocks in a file's lines.
+  module CodeBlocks
+    FENCE = /\A {0,3}(`{3,}|~{3,})[ \t]*(\S*)/
+    SNIPPET_LANGUAGES = %w[sh bash shell zsh console curl].freeze
+
+    module_function
+
+    # The block opening at index, closed by a fence of the same character that's at least as long.
+    def at(lines, index)
+      fence, language = lines[index]&.match(FENCE)&.captures
+      return unless fence
+
+      close = (index + 1...lines.length).find { |i| lines[i].match?(/\A {0,3}#{fence[0]}{#{fence.length},}[ \t]*$/) }
+      { open: index, close:, language:, body: lines[index + 1...close].join } if close
+    end
+
+    # An endpoint's curl snippet and its response, if it has one yet.
+    # A second snippet straight after is another example rather than the response.
+    def endpoint(lines, index)
+      snippet, response = following(lines, index, limit: 2)
+      response && snippet?(response) ? [snippet] : [snippet, response].compact
+    end
+
+    def snippet?(block)
+      SNIPPET_LANGUAGES.include?(block[:language].downcase) || block[:body].lstrip.start_with?('curl')
+    end
+
+    # Up to limit blocks from index on, with only blank lines before and between them.
+    def following(lines, index, limit:)
+      blocks = []
+
+      while blocks.length < limit && (block = at(lines, skip_blank_lines(lines, index)))
+        blocks << block
+        index = block[:close] + 1
+      end
+
+      blocks
+    end
+
+    def skip_blank_lines(lines, index)
+      index += 1 while lines[index]&.strip&.empty?
+      index
+    end
+  end
+end

data/lib/raindown/config.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Raindown
+  # A project's settings, from config.yml in its .raindown folder:
+  #   hide_directives: true    # Directives become HTML comments rather than bold text in the HTML.
+  module Config
+    FILE = 'config.yml'
+    DEFAULTS = { hide_directives: false }.freeze
+
+    module_function
+
+    # The settings for the project a file is in, or the defaults outside one.
+    def for(path)
+      folder = Nodes.find_folder(File.dirname(File.expand_path(path)))
+      file = folder && File.join(folder, FILE)
+      return DEFAULTS unless file && File.exist?(file)
+
+      DEFAULTS.merge(YAML.safe_load_file(file, symbolize_names: true) || {})
+    end
+  end
+end

data/lib/raindown/directives.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'erb'
+
+module Raindown
+  # Content directives, like a node's directive and end tag or an endpoint's, are kept in the HTML:
+  # in bold by default, or as HTML comments with the hide_directives setting.
+  # Until then each is a comment marker, which ends the Markdown block before it and starts nothing itself,
+  # so a node's list stays its own list.
+  # Template directives, like for: and if:, are always invisible, since they often sit inside tables and lists.
+  module Directives
+    MARKER = /<!-- raindown:directive (.*?) -->/
+
+    module_function
+
+    # A directive like "files: folder: 'notes'", or a whole line like <{ :files }>, as a marker.
+    def marker(directive)
+      text = directive.strip.delete_prefix('<{').delete_suffix('}>').strip.gsub('--', '- -')
+      "<!-- raindown:directive #{text} -->"
+    end
+
+    def render(html, hide: false)
+      html.gsub(MARKER) do
+        text = ::Regexp.last_match(1)
+        hide ? "<!-- #{text} -->" : "<p><strong>#{ERB::Util.html_escape(text)}</strong></p>"
+      end
+    end
+  end
+end

data/lib/raindown/endpoint/curl.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'shellwords'
+
+module Raindown
+  # Runs a curl snippet with the curl binary. There's no shell, so pipes, redirects and variables don't apply.
+  # Snippets come from files that anyone (or any agent) may have written, so only options that make an HTTP request
+  # are allowed: nothing that reads or writes local files, loads a config, or uses another protocol.
+  module Curl
+    class Error < StandardError; end
+
+    TIMEOUT = '30'
+    # --disable must come first and skips ~/.curlrc. The protocol options stop redirects to file:// and the like.
+    SAFE_OPTIONS = ['--disable', '--proto', '=http,https', '--proto-redir', '=http,https'].freeze
+
+    OPERATORS = %w[| || & && ; > >> < 2> 2>&1].freeze
+    URL = %r{\Ahttps?://}i
+
+    # Options without a value.
+    FLAGS = {
+      '-G' => '--get', '-I' => '--head', '-L' => '--location', '-i' => '--include', '-k' => '--insecure',
+      '-f' => '--fail', '-g' => '--globoff', '-s' => '--silent', '-S' => '--show-error'
+    }.freeze
+    LONG_FLAGS = (FLAGS.values + %w[--compressed --http1.1 --http2]).freeze
+
+    # Options with a value, and how the value is checked.
+    VALUE_OPTIONS = {
+      '-X' => '--request', '-H' => '--header', '-d' => '--data', '-A' => '--user-agent', '-e' => '--referer',
+      '-u' => '--user'
+    }.freeze
+    VALUE_CHECKS = {
+      '--request' => :any, '--user-agent' => :any, '--referer' => :any, '--user' => :any, '--data-raw' => :any,
+      '--url' => :url,
+      # "@file" reads a file.
+      '--header' => :no_file, '--data' => :no_file, '--data-ascii' => :no_file, '--data-binary' => :no_file,
+      '--json' => :no_file,
+      # "@file" and "name@file" read a file, but "name=a@b" doesn't.
+      '--data-urlencode' => :no_urlencoded_file
+    }.freeze
+
+    # Separates the response body from the content type that --write-out appends.
+    TYPE_MARKER = "\n--raindown-content-type--"
+    WRITE_OUT = "#{TYPE_MARKER}%{content_type}".freeze # rubocop:disable Style/FormatStringToken -- curl's syntax, not Ruby's
+    LANGUAGES = {
+      /json/ => 'json', /xml/ => 'xml', /html/ => 'html', /javascript/ => 'javascript',
+      /css/ => 'css', /yaml/ => 'yaml', %r{\Atext/plain} => 'text'
+    }.freeze
+
+    module_function
+
+    # Returns the response body and the code block language for its content type.
+    def run(snippet)
+      stdout, stderr, status = Open3.capture3('curl', *SAFE_OPTIONS, '--silent', '--show-error', '--max-time', TIMEOUT,
+                                              *arguments(snippet), '--write-out', WRITE_OUT)
+      raise Error, stderr.strip.empty? ? "curl exited with #{status.exitstatus}" : stderr.strip unless status.success?
+
+      body, _, content_type = stdout.rpartition(TYPE_MARKER)
+      language = language(content_type)
+
+      [language == 'json' ? pretty_json(body) : body, language]
+    rescue Errno::ENOENT
+      raise Error, 'curl is not installed'
+    end
+
+    def arguments(snippet)
+      words = Shellwords.split(snippet.gsub(/\\\r?\n/, ' ').strip.delete_prefix('$ '))
+      raise Error, 'Endpoint snippets must start with curl' unless words.first == 'curl'
+      raise Error, 'Endpoint snippets must be a single curl command' if words.intersect?(OPERATORS)
+
+      check(words.drop(1))
+    rescue ArgumentError => e
+      raise Error, "Can't read the curl snippet: #{e.message}"
+    end
+
+    # Returns the arguments with short options spelled long, or raises on anything not allowed.
+    def check(words)
+      args = []
+      words = words.dup
+
+      until words.empty?
+        word = words.shift
+
+        if word.start_with?('--') then args.concat(long_option(word, words))
+        elsif word.start_with?('-') && word.length > 1 then args.concat(short_options(word, words))
+        else args << url(word)
+        end
+      end
+
+      raise Error, 'Endpoint snippets need a URL' unless args.any? { |arg| arg.match?(URL) }
+
+      args
+    end
+
+    def long_option(option, words)
+      return [option] if LONG_FLAGS.include?(option)
+      raise Error, "curl's #{option} option isn't allowed in endpoint snippets" unless VALUE_CHECKS.key?(option)
+      raise Error, "curl's #{option} option needs a value" if words.empty?
+
+      [option, value(option, words.shift)]
+    end
+
+    # Short flags can be grouped like -sSL, and a short option's value can be attached like -XPOST.
+    def short_options(word, words)
+      letters = word.delete_prefix('-')
+      args = []
+
+      letters.each_char.with_index do |letter, index|
+        option = "-#{letter}"
+        next args << FLAGS[option] if FLAGS.key?(option)
+        raise Error, "curl's #{option} option isn't allowed in endpoint snippets" unless VALUE_OPTIONS.key?(option)
+
+        # The value is the rest of the word, or else the next word.
+        attached = letters[(index + 1)..]
+        words.unshift(attached) unless attached.empty?
+        return args.concat(long_option(VALUE_OPTIONS[option], words))
+      end
+
+      args
+    end
+
+    def value(option, value)
+      case VALUE_CHECKS.fetch(option)
+      when :url then url(value)
+      when :no_file then no_file(option, value, value.start_with?('@'))
+      when :no_urlencoded_file then no_file(option, value, value.match?(/\A[^=]*@/))
+      else value
+      end
+    end
+
+    def no_file(option, value, reads_file)
+      raise Error, "curl's #{option} can't read a file (#{value}) in endpoint snippets" if reads_file
+
+      value
+    end
+
+    def url(value)
+      raise Error, "Endpoint snippets can only request http(s) URLs, not #{value}" unless value.match?(URL)
+
+      value
+    end
+
+    def language(content_type)
+      LANGUAGES.find { |pattern, _| content_type.match?(pattern) }&.last || ''
+    end
+
+    def pretty_json(body)
+      JSON.pretty_generate(JSON.parse(body))
+    rescue JSON::ParserError
+      body
+    end
+  end
+end