fronde 0.3.4 → 0.5.0

data/lib/fronde/source/html.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Fronde
+  class Source
+    # Specific settings for HTML {Fronde::Source}
+    class Html < Source
+      def org_config
+        config = super
+        config[0]['theme'] = @config['theme']
+        config
+      end
+
+      class << self
+        def org_default_postamble
+          <<~POSTAMBLE
+            <p><span class="author">#{R18n.t.fronde.org.postamble.written_by}</span>
+            #{R18n.t.fronde.org.postamble.with_emacs_html}</p>
+            <p class="date">#{R18n.t.fronde.org.postamble.last_modification}</p>
+            <p class="validation">%v</p>
+          POSTAMBLE
+        end
+      end
+
+      private
+
+      def fill_in_specific_config
+        @config.merge!(
+          'type' => 'html', 'ext' => '.html', 'mime_type' => 'text/html',
+          'folder' => CONFIG.get('html_public_folder')
+        )
+      end
+
+      def org_publish_options
+        if @config['is_blog']
+          @config['atom_feed'] = <<~ATOMFEED
+            <link rel="alternate" type="application/atom+xml" title="#{@config['title']}"
+                  href="#{@config['domain']}#{public_absolute_path}feeds/index.xml" />
+          ATOMFEED
+        end
+        super
+      end
+
+      def org_default_options
+        defaults = {
+          'publishing-function' => 'org-html-publish-to-html',
+          'html-head-include-default-style' => 't',
+          'html-head-include-scripts' => 't',
+          'html-head' => '%F',
+          'html-postamble' => Html.org_default_postamble
+        }
+        return defaults if @config['theme'] == 'default'
+
+        defaults.merge(
+          'html-head-include-default-style' => 'nil',
+          'html-head-include-scripts' => 'nil',
+          'html-head' => <<~HTMLHEAD
+            <link rel="stylesheet" type="text/css" media="screen"
+                  href="%h/assets/%o/css/style.css">
+            <link rel="stylesheet" type="text/css" media="screen"
+                  href="%h/assets/%o/css/htmlize.css">
+            %F
+          HTMLHEAD
+        )
+      end
+    end
+  end
+end

data/lib/fronde/source.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+module Fronde
+  # Wrapper for each possible project source.
+  # Must be subclassed by specific source type (like Gemini or HTML)
+  class Source
+    def initialize(source_config)
+      @config = {
+        'recursive' => true, 'is_blog' => false,
+        'domain' => CONFIG.get('domain'), 'atom_feed' => '',
+        'org-options' => {
+          'section-numbers' => 'nil', 'with-toc' => 'nil'
+        }
+      }.merge(source_config)
+      clean_config
+      org_publish_options
+      render_heading
+    end
+
+    def [](key)
+      @config[key]
+    end
+
+    def []=(key, value)
+      @config[key] = value
+    end
+
+    def type
+      @config['type']
+    end
+
+    def recursive?
+      !!@config['recursive']
+    end
+
+    def blog?
+      !!@config['is_blog']
+    end
+
+    def to_h
+      @config
+    end
+
+    def source_for?(file_name)
+      relative_file_path = file_name.delete_prefix "#{@config['path']}/"
+      # file_name does not begin with source path.
+      return false if relative_file_path == file_name
+
+      # Looks like a file at a deeper level, but current source is not
+      # recursive.
+      return false if relative_file_path.include?('/') && !recursive?
+
+      # We don’t check file if the file really exist as the current
+      # check may be done before the file is actually written.
+      true
+    end
+
+    def source_for(file_name)
+      relative_file_path = file_name.delete_prefix "#{publication_path}/"
+      # file_name does not begin with source path.
+      return nil if relative_file_path == file_name
+
+      # Looks like a file at a deeper level, but current source is not
+      # recursive.
+      return nil if relative_file_path.include?('/') && !recursive?
+
+      # Looks like a match. But does a source file for this one actually
+      # exists?
+      relative_source_path = relative_file_path.sub(
+        /#{@config['ext']}\z/, '.org'
+      )
+      source_path = File.join(@config['path'], relative_source_path)
+      return nil unless File.file?(source_path)
+
+      source_path
+    end
+
+    def target_for(file_name)
+      target = File.expand_path file_name
+      target.delete_prefix! "#{Dir.pwd}/"
+      target.sub!(/\.org\z/, @config['ext'])
+      project_relative_path = @config['path'].delete_prefix("#{Dir.pwd}/")
+      target.delete_prefix! "#{project_relative_path}/"
+      public_absolute_path + target
+    end
+
+    def exclude_file?(file_name)
+      # Obviously excluding index itself for blogs
+      return true if file_name == File.join(@config['path'], 'index.org')
+
+      exclusion_rules = @config['exclude']
+      return false unless exclusion_rules
+
+      file_name.match? exclusion_rules
+    end
+
+    def org_config
+      name = @config['name']
+      [{ 'name' => name, 'attributes' => org_project_config },
+       { 'name' => "#{name}-assets", 'attributes' => org_assets_config }]
+    end
+
+    # Return the publication absolute path on file system.
+    #
+    # The returned string never end with a slash (/).
+    #
+    # Use {Fronde::Source#public_absolute_path} to get the absolute path
+    # of this project, as seen from a web browser.
+    #
+    # @return [String] the absolute path to the target dir of this project
+    def publication_path
+      return @config['publication_path'] if @config['publication_path']
+
+      publish_in = [File.expand_path(@config['folder']), @config['target']]
+      @config['publication_path'] = publish_in.join('/').delete_suffix('/')
+    end
+
+    # Return the absolute path as seen in User Agent.
+    #
+    # The returned string always end with a slash (/).
+    #
+    # Use {Fronde::Source#publication_path} to locate published file on
+    # the file system.
+    #
+    # @return [String] the absolute path to this project
+    def public_absolute_path
+      return @config['public_absolute_path'] if @config['public_absolute_path']
+
+      @config['public_absolute_path'] = "/#{@config['target']}/".squeeze('/')
+    end
+
+    class << self
+      def canonical_config(config)
+        config = { 'path' => config } if config.is_a?(String)
+        config['type'] ||= 'html'
+        config
+      end
+
+      def new_from_config(config)
+        klass_name = config['type'].capitalize
+        klass = Kernel.const_get("::Fronde::Source::#{klass_name}")
+        klass.new(config)
+      end
+    end
+
+    private
+
+    def clean_config
+      fill_in_specific_config
+      @config['name'] ||= @config['path'].sub(%r{^[.~]*/}, '').tr('/.', '-')
+      @config['title'] ||= @config['path']
+      @config['target'] ||= File.basename(@config['path']).delete_prefix '.'
+      @config['target'] = '' if @config['target'] == '.'
+      @config['path'] = File.expand_path(@config['path'])
+      @config['theme'] ||= CONFIG.get('theme', 'default')
+      # Blog are necessarily recursive to allow publication of tags and feeds
+      @config['recursive'] = true if @config['is_blog']
+    end
+
+    def org_publish_options
+      type = @config['type']
+      options = org_default_options.merge(
+        @config['org-options'],
+        CONFIG.get("org-#{type}", {}),
+        @config["org-#{type}"] || {}
+      )
+      @config['org-options'] = options
+    end
+
+    def render_heading
+      %w[head head-extra preamble postamble].each do |kind|
+        heading_key = "#{@config['type']}-#{kind}"
+        heading = @config.dig 'org-options', heading_key
+        next unless heading
+
+        @config['org-options'][heading_key] =
+          heading.gsub('%F', @config['atom_feed'])
+                 .gsub('%h', @config['domain'])
+                 .gsub('%o', @config['theme'])
+      end
+    end
+
+    def org_project_config
+      attributes = {
+        'base-directory' => @config['path'],
+        'base-extension' => 'org',
+        'publishing-directory' => publication_path,
+        'recursive' => @config['recursive'],
+        'fronde-base-uri' => "#{@config['domain']}#{public_absolute_path}"
+      }.merge(@config['org-options'])
+      exclude = @config['exclude']
+      attributes['exclude'] = exclude if exclude
+      attributes.sort.to_h # Have lisp config sorted
+    end
+
+    def org_assets_config
+      {
+        'base-directory' => @config['path'],
+        'base-extension' => %w[gif jpg png svg pdf].join('\\\\|'),
+        'publishing-directory' => publication_path,
+        'publishing-function' => 'org-publish-attachment',
+        'recursive' => @config['recursive']
+      }
+    end
+  end
+end
+
+require_relative 'source/gemini'
+require_relative 'source/html'

data/lib/fronde/sync/neocities.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'yaml'
+require 'net/http'
+require 'fileutils'
+require 'digest/sha1'
+
+module Fronde
+  module Sync
+    # Everything needed to connect to neocities
+    class Neocities
+      PROTECTED_FILES = %w[index.html neocities.png not_found.html].freeze
+
+      def initialize(connection_spec, public_folder, verbose: false, &block)
+        @verbose = verbose
+        @endpoint = @website_name = @authorization = nil
+        extract_connection_details connection_spec
+        @public_folder = public_folder
+        @connection = init_connection
+        return unless block
+
+        yield self
+        finish
+      end
+
+      def remote_list
+        remote = call build_request('/list')
+        JSON.parse(remote.body)['files'].map do |stat|
+          stat['updated_at'] = Time.parse(stat['updated_at'])
+          stat
+        end
+      end
+
+      def local_list
+        Dir.chdir(@public_folder) do
+          Dir['**/*'].map { |file| neocities_stat(file) }
+        end
+      end
+
+      def info
+        info = call build_request('/info')
+        JSON.parse(info.body)['info']
+      end
+
+      def finish
+        @connection.finish if @connection.started?
+      end
+
+      def pull(test: false)
+        file_list = remote_list
+        finish
+        orphans = select_orphans(file_list, local_list) do |path|
+          warn "deleting #{path}" if @verbose
+
+          "#{@public_folder}/#{path}"
+        end
+        File.unlink(*orphans) unless test
+        download_all file_list, test: test
+        nil # Mute this method
+      end
+
+      def push(test: false)
+        file_list = local_list
+        remove_remote_orphans file_list, test: test
+        upload_all file_list, test: test
+        finish
+      end
+
+      private
+
+      def neocities_stat(file)
+        stat = File.stat(file)
+        data = {
+          'path' => file,
+          'is_directory' => stat.directory?,
+          'updated_at' => stat.mtime.round.utc
+        }
+        return data if data['is_directory']
+
+        data['size'] = stat.size
+        data['sha1_hash'] = Digest::SHA1.hexdigest File.read(file)
+        data
+      end
+
+      def select_orphans(to_apply, current_list, &block)
+        paths_to_apply = to_apply.map { _1['path'] }
+        current_paths = current_list.map { _1['path'] }
+        (current_paths - paths_to_apply).filter_map(&block)
+      end
+
+      def remove_remote_orphans(file_list, test: false)
+        request = build_request '/delete', :post
+        orphan_paths = select_orphans(file_list, remote_list) do |path|
+          # Never remove the following files. If needed you can still
+          # overwrite them. And in any case, neocities will not allow
+          # the index.html file to be removed.
+          next if PROTECTED_FILES.include? path
+
+          warn "deleting #{path}" if @verbose
+          path
+        end
+        request.form_data = { 'filenames[]' => orphan_paths }
+        return if test
+
+        call request
+      end
+
+      def download_all(file_list, test: false)
+        publish_domain = "#{@website_name}.#{@endpoint.host}"
+        Dir.chdir(@public_folder) do
+          Net::HTTP.start(publish_domain, use_ssl: true) do |http|
+            file_list.each do |file_data|
+              path = file_data['path']
+              file_data['uri'] = "https://#{publish_domain}/#{path}"
+              download_file http, file_data, test: test
+            end
+          end
+        end
+      end
+
+      def download_file(http, file_data, test: false)
+        path = file_data['path']
+        if file_data['is_directory']
+          warn "#{path}/" if @verbose
+          FileUtils.mkdir_p path unless test
+          return
+        end
+
+        warn path if @verbose
+
+        content = fetch_file_content(
+          http, file_data['uri'], file_data['sha1_hash']
+        )
+        return unless content && !test
+
+        save_file path, content, file_data['updated_at']
+      end
+
+      def fetch_file_content(http, uri, sha1sum)
+        # Neocities redirect HTML file to location without extension and
+        # redirect index.html file to /
+        uri = uri.delete_suffix('index.html').delete_suffix('.html')
+        content = http.get(uri).body
+        check = Digest::SHA1.hexdigest content
+        return content if check == sha1sum
+
+        warn "SHA1 hash differ for #{uri}"
+      end
+
+      def save_file(path, content, updated_at)
+        File.write path, content
+        FileUtils.touch path, mtime: updated_at
+        path
+      end
+
+      def prepare_files_to_upload(file_list)
+        Dir.chdir(@public_folder) do
+          file_list.filter_map do |file_data|
+            # No need to push intermediary directories, they are created
+            # on the fly
+            next if file_data['is_directory']
+
+            path = file_data['path']
+            warn path if @verbose
+            [path, File.new(path)]
+          end
+        end
+      end
+
+      def upload_all(file_list, test: false)
+        form_data = prepare_files_to_upload file_list
+        return if test
+
+        request = build_request '/upload', :post
+        request.set_form form_data, 'multipart/form-data'
+        call request
+      end
+
+      def extract_connection_details(connection_spec)
+        # Do not put your password into the fronde config. The password
+        # is expectfed to be found in a specific config file (not to be
+        # shared).
+        @website_name, endpoint = connection_spec.split('@', 2)
+        endpoint ||= 'neocities.org'
+        @endpoint = URI("https://#{endpoint}/api")
+        # Will raise Errno::ENOENT if file does not exist
+        credentials = YAML.load_file('.credentials')
+        # Will raise KeyError if not set
+        password = credentials.fetch("#{@website_name}_neocities_pass")
+        authorization = [[@website_name, password].join(':')].pack('m0')
+        @authorization = "Basic #{authorization}"
+      end
+
+      def build_request(path, method = :get)
+        uri = @endpoint.dup
+        uri.path += path
+        klass = Kernel.const_get "Net::HTTP::#{method.to_s.capitalize}"
+        klass.new uri
+      end
+
+      def call(request)
+        request['Authorization'] = @authorization
+        @connection.start unless @connection.started?
+        outcome = @connection.request request
+        return outcome if outcome.is_a? Net::HTTPSuccess
+
+        raise JSON.parse(outcome.body).inspect
+      end
+
+      def init_connection
+        http = Net::HTTP.new @endpoint.host, 443
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http
+      end
+    end
+  end
+end

data/lib/fronde/sync/rsync.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative '../config'
+
+module Fronde
+  module Sync
+    # Everything needed to push or pull data with rsync
+    class Rsync
+      def initialize(remote_path, local_path, verbose: false)
+        @verbose = verbose
+        @remote_path = remote_path
+        @local_path = "#{local_path}/"
+      end
+
+      def pull(test: false)
+        run command(test: test) + [@remote_path, @local_path]
+      end
+
+      def push(test: false)
+        run command(test: test) + [@local_path, @remote_path]
+      end
+
+      private
+
+      def run(cmd)
+        warn cmd.join(' ') if @verbose
+        # Be precise about Kernel to allow mock in rspec
+        Kernel.system(*cmd)
+      end
+
+      def command(test: false)
+        rsync_command = Fronde::CONFIG.get('rsync')
+        return rsync_command unless rsync_command.nil?
+
+        optstring = []
+        optstring << 'n' if test
+        if @verbose
+          optstring << 'v'
+        else
+          optstring << 'q'
+        end
+        ['rsync', "-#{optstring.join}rlt", '--delete']
+      end
+    end
+  end
+end

data/lib/fronde/sync.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'config'
+require_relative 'sync/rsync'
+require_relative 'sync/neocities'
+
+module Fronde
+  # Entrypoint for synchronization with remote public server
+  module Sync
+    class Error < ::StandardError; end
+
+    ALLOWED_SYNCER = %w[rsync neocities].freeze
+
+    def self.extract_method_and_remote(type)
+      remote_path = Fronde::CONFIG.get("#{type}_remote")
+      raise Error, "No #{type} remote path set" if remote_path.nil?
+
+      method, remote = remote_path.split(':', 2)
+      return [method, remote] if ALLOWED_SYNCER.include?(method)
+
+      ['rsync', remote_path]
+    end
+
+    def self.pull_or_push(direction, type, test: false, verbose: false)
+      method, remote_path = extract_method_and_remote type
+      public_folder = Fronde::CONFIG.get("#{type}_public_folder")
+      klass = Kernel.const_get("::Fronde::Sync::#{method.capitalize}")
+      syncer = klass.new(remote_path, public_folder, verbose: verbose)
+      Thread.new { syncer.send(direction, test: test) }
+    end
+  end
+end