raindown 0.16.0 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b35949f91c9cf54ccb1ab0fe6b0e24a2e9529f830cd648f097015aba55acb135
-  data.tar.gz: 3b62d3623ea255e6767b349a3210c48a331bcdae8282d9a365adea680e2242e8
+  metadata.gz: 2abf28897b1c392d34ec786d7abad550414050fe3f45949929240fa31453cf23
+  data.tar.gz: edafdc330bddeaf1b76a4d6538b1e3db9241c1758526d0076b0ffe76d5aac32a
 SHA512:
-  metadata.gz: c63653585217df9c3129b554f0619fe024916a73c7eaea2fcad53199ee2f790fbece26bf90c956fa716109080b6c457048acbc3bb39fa97d3520765b4d9ff47d
-  data.tar.gz: 1f6e1d91403fcfbf9f2968885bf94a227a1a114084a65bb8735c7dbdff4479b622f8fa3ea17fab42f7e3cc826372c726d1d90f137b50eac32a92f531b935f20a
+  metadata.gz: 5ce17ac9171dfa2d3a37311698f53e89390eccd2d3105a60adef58136f6bf4f741162ecd6eea0edc10a341434aaa58b1b8832e1971cc1ae3498771255616e3ce
+  data.tar.gz: 15ff81a860a3403f8bca98ca63052be5d8ff1cc9feda5da235f6b753ffd92f74ba313471ac60d7b61c59fb7c84f38bd05e8dd7dfa68877a66d84bd5251499217

data/lib/raindown/chart/points.rb

@@ -5,6 +5,7 @@ require 'json'
 require 'yaml'
 
 require_relative '../node'
+require_relative '../utils'
 
 module Raindown
   class Chart < Node
@@ -43,6 +44,7 @@ module Raindown
         return numbers(numbers) if numbers.any? && numbers.all? { number(it) }
 
         path = File.expand_path(string, @root)
+        raise Error, "it's outside the project" unless Utils.inside?(path, @root)
         raise Error, 'no such file' unless File.file?(path)
 
         from(read(path))

data/lib/raindown/cli.rb

@@ -10,6 +10,7 @@ require_relative 'blueprints'
 require_relative 'editing'
 require_relative 'rewriter'
 require_relative 'server'
+require_relative 'skeleton'
 require_relative 'watcher/file_watcher'
 require_relative 'watcher/supervisor'
 
@@ -32,6 +33,8 @@ module Raindown
                             PATH is a file, or a directory (default: current directory) whose projects with a
                             .raindown folder are watched
         init [DIRECTORY]    Create a .raindown folder for the project's config and nodes (default: current directory)
+        skeleton            Set this repo up to commit directives without what they write, in files with "skeleton: true"
+        skeleton --check    Exit with 1, naming any such file with output staged, for CI
         server              Keep Raindown loaded for an editor, answering a JSON request a line on stdin with its HTML,
                             or its Markdown as it would be written back, like --stdin and --stdin --rewrite
         blueprints            Save the blueprints in Claude Code's last reply to .raindown/data/blueprints.md, for the terminal
@@ -78,17 +81,22 @@ module Raindown
     def self.run(argv)
       # Markdown is UTF-8 whatever the locale, as an app opened from the Dock, like Obsidian, starts Raindown without one.
       Encoding.default_external = Encoding::UTF_8
-      return if command(argv)
-
-      # Without a path, like "raindown", "raindown -x" or "raindown watch", it's the current directory.
-      argv = ['watch', '.', *argv.drop(1)] if argv.first == 'watch' && (argv.size == 1 || argv[1].start_with?('-'))
-      argv = ['.', *argv] if argv.empty? || argv.first.start_with?('-')
+      return if command(argv) || Skeleton.run_command(argv)
 
+      argv = with_path(argv)
       super || abort("Unknown arguments: #{argv.join(' ')}\n\n#{USAGE}")
     rescue StandardError => e
       report(e, path: argv.first)
     end
 
+    # Without a path, like "raindown", "raindown -x" or "raindown watch", it's the current directory.
+    def self.with_path(argv)
+      return ['watch', '.', *argv.drop(1)] if argv.first == 'watch' && (argv.size == 1 || argv[1].start_with?('-'))
+      return ['.', *argv] if argv.empty? || argv.first.start_with?('-')
+
+      argv
+    end
+
     # The :path line would match help, version and init as the file, so they're checked first.
     # Returns true when argv was one of them.
     def self.command(argv)

data/lib/raindown/front_matter.rb

@@ -3,6 +3,8 @@
 require 'date'
 require 'yaml'
 
+require_relative 'utils'
+
 module Raindown
   # YAML between two --- lines at the very start of a file.
   module FrontMatter
@@ -19,6 +21,14 @@ module Raindown
       [metadata.is_a?(Hash) ? metadata : {}, match.post_match]
     end
 
+    # A file's metadata, or none when it can't be read, with a warning naming it, so one file can't stop a list of them.
+    def read(file_path)
+      parse(Utils.read_markdown(file_path)).first
+    rescue Psych::Exception, ArgumentError => e
+      warn "raindown: #{file_path}: #{e.message}, so it's listed without its front matter"
+      {}
+    end
+
     # How many of the lines are front matter, so directives in its YAML are left alone.
     def line_count(lines)
       lines.join.match(PATTERN)&.then { |match| match[0].lines.length } || 0

data/lib/raindown/list.rb

@@ -14,18 +14,26 @@ module Raindown
       @tags = tags
     end
 
-    # Each file has its front matter, its path and its content. Sorted by an order, files without it come last, and ones
-    # with the same value stay in the order they're listed.
+    # Each file has its front matter, its path and its content. Sorted by an order, numbers come first, then other values
+    # as text, then files without it, and ones with the same value stay in the order they're listed.
     def files(metadata: NoMetadata)
       files = metadata.list(**@tags)
       return files unless @order
 
       sorted, unsorted = files.partition { it[@order] }
-      sorted.sort_by.with_index { |file, index| [file[@order], index] } + unsorted
+      sorted.sort_by.with_index { |file, index| [*order_key(file[@order]), index] } + unsorted
     end
 
     def rewrite(metadata: NoMetadata)
-      files(metadata:).map { |file| "- [#{file.title || file.path}](#{file.path})" }.join("\n")
+      files(metadata:).map { |file| "- [#{file.title || File.basename(file.path, '.*')}](#{file.path})" }.join("\n")
+    end
+
+    private
+
+    def order_key(value)
+      return [0, value, ''] if value.is_a?(Numeric)
+
+      [1, 0, value.to_s]
     end
   end
 end

data/lib/raindown/pages.rb

@@ -25,10 +25,11 @@ module Raindown
         @front_matters ||= {}
       end
 
-      # The Markdown files under a project's root, the folder with its .raindown folder or else the file's folder.
+      # The Markdown files under a project's root, the folder with its .raindown folder, or a vault, or else the file's
+      # folder.
       def for(path)
-        file_directory = File.dirname(File.expand_path(path))
-        load(root: Nodes.find_folder(file_directory)&.then { File.dirname(it) } || file_directory)
+        require_relative 'projects'
+        load(root: Projects.content_root(path) || File.dirname(File.expand_path(path)))
       end
 
       # Lists the files under the root, without loading them, for Dirload.
@@ -118,7 +119,7 @@ module Raindown
       modified_then, front_matter = self.class.front_matters[file_path]
       return front_matter if modified_then == modified
 
-      FrontMatter.parse(Utils.read_markdown(file_path)).first.tap { self.class.front_matters[file_path] = [modified, it] }
+      FrontMatter.read(file_path).tap { self.class.front_matters[file_path] = [modified, it] }
     end
 
     # The path is relative to the root, like docs/_basics/_1-getting-started.md.

data/lib/raindown/projects.rb

@@ -44,6 +44,18 @@ module Raindown
       folder && File.dirname(folder)
     end
 
+    # The folder a file's paths are relative to: the nearest with a .raindown folder or that's a vault, or nil without one.
+    # A vault doesn't need a .raindown folder, as Obsidian's links and a table's path are from its top.
+    def content_root(path)
+      folder = File.dirname(File.expand_path(path))
+      until project?(folder) || vault?(folder)
+        return if folder == File.dirname(folder)
+
+        folder = File.dirname(folder)
+      end
+      folder
+    end
+
     # The project's Markdown files, sorted, leaving out ignored folders. The projects inside it are left out too,
     # since they have nodes of their own, and vaults, which their editor writes back to.
     def files(root)

data/lib/raindown/skeleton.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+require_relative 'bare_directives'
+require_relative 'document'
+require_relative 'front_matter'
+require_relative 'nodes'
+require_relative 'pages'
+require_relative 'utils'
+
+module Raindown
+  # A file with "skeleton: true" in its front matter keeps its directives in git, but not what they write, so dashboards
+  # don't fill its history and sensitive reports stay out of it. It's a git clean filter: the file on disk keeps its
+  # output, and what's committed doesn't, which a clone writes again when it's rendered. The filter's required, so when
+  # a file can't be stripped, git stops rather than commit its output.
+  module Skeleton
+    FILTER = 'raindown'
+    SKELETON = /^skeleton:[ \t]*true[ \t]*$/
+
+    module_function
+
+    # Returns true when the arguments were "raindown skeleton", with --check or a file or neither.
+    def run_command(argv)
+      return false unless argv.first == 'skeleton' && argv.size <= 2
+
+      run(argv[1])
+      true
+    end
+
+    # "raindown skeleton" sets the repo up, --check is for CI, and with a file it's git's filter.
+    def run(argument)
+      root, = Open3.capture2('git', 'rev-parse', '--show-toplevel')
+      abort('raindown skeleton needs a git repo') if root.strip.empty?
+      return install(root.strip) unless argument
+      return check(root.strip) if argument == '--check'
+
+      filter(argument)
+    end
+
+    # From stdin to stdout, which it has to itself, so what nodes print goes to stderr, like the server's answers.
+    def filter(path)
+      output = $stdout.dup
+      $stdout.reopen($stderr)
+      output.write(strip($stdin.read, path:))
+    end
+
+    def check(root)
+      files = staged_output(root)
+      return if files.empty?
+
+      warn('Output is staged in these skeleton files. Run "raindown skeleton", then "git add --renormalize .":', *files)
+      exit 1
+    end
+
+    # The Markdown without what its directives wrote, or as it is, to the byte, without "skeleton: true".
+    def strip(markdown, path:)
+      return markdown unless skeleton?(markdown)
+
+      FrontMatter.parse(markdown) # Raises when it can't be read, as bare directives would be missed.
+      Nodes.load(path)
+      markdown = markdown.gsub("\r\n", "\n")
+      lines = markdown.lines
+      outputs(markdown, path).reverse_each { |output| lines.slice!((output.begin - 1)..(output.end - 1)) }
+      lines.join
+    end
+
+    # Sets the filter up for a repo. Git keeps a filter's command in its own config, so each clone needs this.
+    def install(root)
+      { clean: "#{FILTER} skeleton %f", smudge: 'cat', required: 'true' }.each do |name, value|
+        system('git', 'config', "filter.#{FILTER}.#{name}", value, chdir: root, exception: true)
+      end
+      add_attributes(File.join(root, '.gitattributes'))
+    end
+
+    # The staged files that still have output, like ones added before the filter was set up, or in a clone without it.
+    def staged_output(root)
+      tracked(root).reject do |path|
+        staged = staged_markdown(root, path)
+        strip(staged, path: File.join(root, path)) == staged
+      end
+    end
+
+    # Read from the front matter's text, so a file with front matter that isn't valid YAML, or isn't UTF-8, can still be
+    # committed when it isn't a skeleton.
+    def skeleton?(markdown)
+      markdown.scrub.match(FrontMatter::PATTERN)&.[](1)&.match?(SKELETON) || false
+    end
+
+    def outputs(markdown, path)
+      bare = BareDirectives.enabled?(path:, content: markdown)
+      Document.new(markdown:, bare:).directives.select(&:writes).filter_map(&:output)
+    end
+
+    def add_attributes(path)
+      existing = File.exist?(path) ? File.read(path) : ''
+      missing = Pages::EXTENSIONS.map { "*.#{it} filter=#{FILTER}" }.reject { existing.include?(it) }
+      return if missing.empty?
+
+      existing += "\n" unless existing.empty? || existing.end_with?("\n")
+      File.write(path, "#{existing}#{missing.join("\n")}\n")
+    end
+
+    def tracked(root)
+      output, = Open3.capture2('git', 'ls-files', '-z', chdir: root)
+      output.split("\0").select { Pages::EXTENSIONS.include?(File.extname(it).delete_prefix('.').downcase) }
+    end
+
+    def staged_markdown(root, path)
+      output, = Open3.capture2('git', 'show', ":#{path}", chdir: root)
+      output.force_encoding(Encoding::UTF_8)
+    end
+  end
+end

data/lib/raindown/table/markdown_table.rb

@@ -35,7 +35,12 @@ module Raindown
     end
 
     def escape(cell)
-      cell.gsub('|', '\|').gsub(/\s*\R\s*/, ' ')
+      one_line(cell).gsub('|', '\|')
+    end
+
+    # The cell as a table shows it, on one line.
+    def one_line(cell)
+      cell.gsub(/\s*\R\s*/, ' ')
     end
   end
 end

data/lib/raindown/table/table.rb

@@ -5,6 +5,7 @@ require 'digest'
 require 'json'
 
 require_relative '../node'
+require_relative '../utils'
 require_relative 'markdown_table'
 require_relative 'table_formulas'
 require_relative 'written_table'
@@ -36,24 +37,31 @@ module Raindown
     # Written is what it wrote last time, so it can tell which of the table and the file changed since.
     def rewrite(written: [], metadata: NoMetadata)
       path = data_path(metadata)
-      data_error(path) || synced(path, written.map(&:chomp))
+      data_error(path, metadata) || synced(path, written.map(&:chomp))
     end
 
     # Write-in, in <{ }>, the table only shows its file, and never writes to it.
     def render(metadata: NoMetadata)
       path = data_path(metadata)
-      Raindown.commonmark(data_error(path) || MarkdownTable.format(with_total(TableFormulas.evaluate(read_rows(path)))))
+      Raindown.commonmark(data_error(path, metadata) || MarkdownTable.format(with_total(TableFormulas.evaluate(read_rows(path)))))
     end
 
     private
 
     def data_path(metadata)
-      File.expand_path(@data.to_s, metadata.respond_to?(:root) ? metadata.root : Dir.pwd)
+      File.expand_path(@data.to_s, data_root(metadata))
+    end
+
+    def data_root(metadata)
+      return metadata.root if metadata.respond_to?(:root)
+
+      Dir.pwd
     end
 
     # Why the file can't be a table, or nil when it can.
-    def data_error(path)
+    def data_error(path, metadata)
       return "Can't show #{@data.inspect} as a table: only CSV and TSV files can be" unless SEPARATORS.key?(File.extname(path).downcase)
+      return "Can't show #{@data.inspect} as a table: it's outside the project" unless Utils.inside?(path, data_root(metadata))
 
       "Can't show #{@data.inspect} as a table: no such file" unless File.file?(path)
     end
@@ -64,7 +72,7 @@ module Raindown
       synced_hash = written.first&.[](COMMENT, 1)
       table = WrittenTable.new(written_rows(written), file)
       return markdown(file, table.revealed) if unchanged?(table.rows, synced_hash, shown)
-      return markdown(write_rows(path, with_formulas(table.rows, file, shown)), table.revealed) if hash_rows(shown) == synced_hash
+      return markdown(write_rows(path, with_file_cells(table.rows, file, shown)), table.revealed) if hash_rows(shown) == synced_hash
 
       warn "raindown: #{@data} and its table both changed, so neither is written until they match"
       ["<!-- table conflict · #{synced_hash} -->", *written.drop(1)].join("\n")
@@ -95,12 +103,13 @@ module Raindown
       revealed.each_with_object(values.map(&:dup)) { |(row, column), shown| shown[row][column] = rows[row][column] }
     end
 
-    # The table's rows, keeping the file's formula in each cell still showing its value.
-    def with_formulas(table, file, shown)
+    # The table's rows, keeping the file's cell where the table still shows it, like a formula as its value, or a cell
+    # with line breaks on one line.
+    def with_file_cells(table, file, shown)
       table.each_with_index.map do |row, row_index|
         row.each_with_index.map do |cell, column|
-          formula = file.dig(row_index, column)
-          next formula if TableFormulas.formula?(formula) && shown.dig(row_index, column) == cell
+          kept = file.dig(row_index, column)
+          next kept if kept && MarkdownTable.one_line(shown.dig(row_index, column).to_s) == cell
 
           cell
         end
@@ -146,8 +155,9 @@ module Raindown
       rows
     end
 
+    # As the table shows them, so a cell with line breaks hashes the same in both.
     def hash_rows(rows)
-      Digest::SHA256.hexdigest(JSON.generate(rows))[0, 6]
+      Digest::SHA256.hexdigest(JSON.generate(rows.map { |row| row.map { MarkdownTable.one_line(it.to_s) } }))[0, 6]
     end
   end
 end

data/lib/raindown/utils.rb

@@ -4,6 +4,11 @@ module Raindown
   module Utils
     module_function
 
+    # Whether a path is below a folder, so a note can't reach a file outside its project, like '../../secrets.csv'.
+    def inside?(path, folder)
+      File.expand_path(path).start_with?(File.join(File.expand_path(folder), ''))
+    end
+
     # A Markdown file's text, with its line endings as LF, "\n", whatever they are in the file.
     #
     # Windows ends lines with CRLF, "\r\n", and Git for Windows checks files out that way, so a file written or cloned on

data/lib/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Raindown
-  VERSION = '0.16.0'
+  VERSION = '0.17.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: raindown
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.0
 platform: ruby
 authors:
 - maedi
@@ -251,6 +251,7 @@ files:
 - lib/raindown/projects.rb
 - lib/raindown/rewriter.rb
 - lib/raindown/server.rb
+- lib/raindown/skeleton.rb
 - lib/raindown/stat.rb
 - lib/raindown/table/markdown_table.rb
 - lib/raindown/table/table.rb