fronde 0.4.0 → 0.5.0

data/lib/fronde/sync/neocities.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'yaml'
+require 'net/http'
+require 'fileutils'
+require 'digest/sha1'
+
+module Fronde
+  module Sync
+    # Everything needed to connect to neocities
+    class Neocities
+      PROTECTED_FILES = %w[index.html neocities.png not_found.html].freeze
+
+      def initialize(connection_spec, public_folder, verbose: false, &block)
+        @verbose = verbose
+        @endpoint = @website_name = @authorization = nil
+        extract_connection_details connection_spec
+        @public_folder = public_folder
+        @connection = init_connection
+        return unless block
+
+        yield self
+        finish
+      end
+
+      def remote_list
+        remote = call build_request('/list')
+        JSON.parse(remote.body)['files'].map do |stat|
+          stat['updated_at'] = Time.parse(stat['updated_at'])
+          stat
+        end
+      end
+
+      def local_list
+        Dir.chdir(@public_folder) do
+          Dir['**/*'].map { |file| neocities_stat(file) }
+        end
+      end
+
+      def info
+        info = call build_request('/info')
+        JSON.parse(info.body)['info']
+      end
+
+      def finish
+        @connection.finish if @connection.started?
+      end
+
+      def pull(test: false)
+        file_list = remote_list
+        finish
+        orphans = select_orphans(file_list, local_list) do |path|
+          warn "deleting #{path}" if @verbose
+
+          "#{@public_folder}/#{path}"
+        end
+        File.unlink(*orphans) unless test
+        download_all file_list, test: test
+        nil # Mute this method
+      end
+
+      def push(test: false)
+        file_list = local_list
+        remove_remote_orphans file_list, test: test
+        upload_all file_list, test: test
+        finish
+      end
+
+      private
+
+      def neocities_stat(file)
+        stat = File.stat(file)
+        data = {
+          'path' => file,
+          'is_directory' => stat.directory?,
+          'updated_at' => stat.mtime.round.utc
+        }
+        return data if data['is_directory']
+
+        data['size'] = stat.size
+        data['sha1_hash'] = Digest::SHA1.hexdigest File.read(file)
+        data
+      end
+
+      def select_orphans(to_apply, current_list, &block)
+        paths_to_apply = to_apply.map { _1['path'] }
+        current_paths = current_list.map { _1['path'] }
+        (current_paths - paths_to_apply).filter_map(&block)
+      end
+
+      def remove_remote_orphans(file_list, test: false)
+        request = build_request '/delete', :post
+        orphan_paths = select_orphans(file_list, remote_list) do |path|
+          # Never remove the following files. If needed you can still
+          # overwrite them. And in any case, neocities will not allow
+          # the index.html file to be removed.
+          next if PROTECTED_FILES.include? path
+
+          warn "deleting #{path}" if @verbose
+          path
+        end
+        request.form_data = { 'filenames[]' => orphan_paths }
+        return if test
+
+        call request
+      end
+
+      def download_all(file_list, test: false)
+        publish_domain = "#{@website_name}.#{@endpoint.host}"
+        Dir.chdir(@public_folder) do
+          Net::HTTP.start(publish_domain, use_ssl: true) do |http|
+            file_list.each do |file_data|
+              path = file_data['path']
+              file_data['uri'] = "https://#{publish_domain}/#{path}"
+              download_file http, file_data, test: test
+            end
+          end
+        end
+      end
+
+      def download_file(http, file_data, test: false)
+        path = file_data['path']
+        if file_data['is_directory']
+          warn "#{path}/" if @verbose
+          FileUtils.mkdir_p path unless test
+          return
+        end
+
+        warn path if @verbose
+
+        content = fetch_file_content(
+          http, file_data['uri'], file_data['sha1_hash']
+        )
+        return unless content && !test
+
+        save_file path, content, file_data['updated_at']
+      end
+
+      def fetch_file_content(http, uri, sha1sum)
+        # Neocities redirect HTML file to location without extension and
+        # redirect index.html file to /
+        uri = uri.delete_suffix('index.html').delete_suffix('.html')
+        content = http.get(uri).body
+        check = Digest::SHA1.hexdigest content
+        return content if check == sha1sum
+
+        warn "SHA1 hash differ for #{uri}"
+      end
+
+      def save_file(path, content, updated_at)
+        File.write path, content
+        FileUtils.touch path, mtime: updated_at
+        path
+      end
+
+      def prepare_files_to_upload(file_list)
+        Dir.chdir(@public_folder) do
+          file_list.filter_map do |file_data|
+            # No need to push intermediary directories, they are created
+            # on the fly
+            next if file_data['is_directory']
+
+            path = file_data['path']
+            warn path if @verbose
+            [path, File.new(path)]
+          end
+        end
+      end
+
+      def upload_all(file_list, test: false)
+        form_data = prepare_files_to_upload file_list
+        return if test
+
+        request = build_request '/upload', :post
+        request.set_form form_data, 'multipart/form-data'
+        call request
+      end
+
+      def extract_connection_details(connection_spec)
+        # Do not put your password into the fronde config. The password
+        # is expectfed to be found in a specific config file (not to be
+        # shared).
+        @website_name, endpoint = connection_spec.split('@', 2)
+        endpoint ||= 'neocities.org'
+        @endpoint = URI("https://#{endpoint}/api")
+        # Will raise Errno::ENOENT if file does not exist
+        credentials = YAML.load_file('.credentials')
+        # Will raise KeyError if not set
+        password = credentials.fetch("#{@website_name}_neocities_pass")
+        authorization = [[@website_name, password].join(':')].pack('m0')
+        @authorization = "Basic #{authorization}"
+      end
+
+      def build_request(path, method = :get)
+        uri = @endpoint.dup
+        uri.path += path
+        klass = Kernel.const_get "Net::HTTP::#{method.to_s.capitalize}"
+        klass.new uri
+      end
+
+      def call(request)
+        request['Authorization'] = @authorization
+        @connection.start unless @connection.started?
+        outcome = @connection.request request
+        return outcome if outcome.is_a? Net::HTTPSuccess
+
+        raise JSON.parse(outcome.body).inspect
+      end
+
+      def init_connection
+        http = Net::HTTP.new @endpoint.host, 443
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        http
+      end
+    end
+  end
+end

data/lib/fronde/sync/rsync.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative '../config'
+
+module Fronde
+  module Sync
+    # Everything needed to push or pull data with rsync
+    class Rsync
+      def initialize(remote_path, local_path, verbose: false)
+        @verbose = verbose
+        @remote_path = remote_path
+        @local_path = "#{local_path}/"
+      end
+
+      def pull(test: false)
+        run command(test: test) + [@remote_path, @local_path]
+      end
+
+      def push(test: false)
+        run command(test: test) + [@local_path, @remote_path]
+      end
+
+      private
+
+      def run(cmd)
+        warn cmd.join(' ') if @verbose
+        # Be precise about Kernel to allow mock in rspec
+        Kernel.system(*cmd)
+      end
+
+      def command(test: false)
+        rsync_command = Fronde::CONFIG.get('rsync')
+        return rsync_command unless rsync_command.nil?
+
+        optstring = []
+        optstring << 'n' if test
+        if @verbose
+          optstring << 'v'
+        else
+          optstring << 'q'
+        end
+        ['rsync', "-#{optstring.join}rlt", '--delete']
+      end
+    end
+  end
+end

data/lib/fronde/sync.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'config'
+require_relative 'sync/rsync'
+require_relative 'sync/neocities'
+
+module Fronde
+  # Entrypoint for synchronization with remote public server
+  module Sync
+    class Error < ::StandardError; end
+
+    ALLOWED_SYNCER = %w[rsync neocities].freeze
+
+    def self.extract_method_and_remote(type)
+      remote_path = Fronde::CONFIG.get("#{type}_remote")
+      raise Error, "No #{type} remote path set" if remote_path.nil?
+
+      method, remote = remote_path.split(':', 2)
+      return [method, remote] if ALLOWED_SYNCER.include?(method)
+
+      ['rsync', remote_path]
+    end
+
+    def self.pull_or_push(direction, type, test: false, verbose: false)
+      method, remote_path = extract_method_and_remote type
+      public_folder = Fronde::CONFIG.get("#{type}_public_folder")
+      klass = Kernel.const_get("::Fronde::Sync::#{method.capitalize}")
+      syncer = klass.new(remote_path, public_folder, verbose: verbose)
+      Thread.new { syncer.send(direction, test: test) }
+    end
+  end
+end

data/lib/fronde/templater.rb

@@ -103,6 +103,18 @@ module Fronde
       end
     end
 
+    def warn_no_element(source)
+      pub_folder = Fronde::CONFIG.get('html_public_folder').sub(
+        /^#{Dir.pwd}/, '.'
+      )
+      warn(
+        R18n.t.fronde.error.templater.no_element_found(
+          source: source, file: "#{pub_folder}#{@org_file.pub_file}"
+        )
+      )
+      '' # Return empty string
+    end
+
     def extract_content
       # We must either have a source or a content key
       source = @config.delete 'source'
@@ -111,19 +123,14 @@ module Fronde
       end
 
       node = @dom.css(source)
-      # Do nothing if we don’t have a reliable content to work with
-      unless node.any?
-        warn(
-          R18n.t.fronde.error.templater.no_element_found(
-            source: source,
-            file: Fronde::CONFIG.get('html_public_folder') + @org_file.pub_file
-          )
-        )
-        return ''
+      if node.any?
+        # Put it back in config
+        @config['source'] = node
+        return node.to_s
       end
 
-      @config['source'] = node
-      node.to_s
+      # Do nothing if we don’t have a reliable content to work with
+      warn_no_element source
     end
 
     def check_path(file_name)

data/lib/fronde/version.rb

@@ -2,5 +2,5 @@
 
 module Fronde
   # @return [String] the version number of the current Fronde release.
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/lib/tasks/org.rake

@@ -8,8 +8,7 @@ require_relative '../fronde/cli/throbber'
 require 'rake/clean'
 
 CLOBBER.push(
-  'var/lib/org-config.el', '.dir-locals.el',
-  'lib/htmlize.el'
+  'var/lib/org-config.el', 'lib/htmlize.el'
 )
 
 namespace :org do
@@ -26,19 +25,16 @@ namespace :org do
     else
       Fronde::CLI::Throbber.run(download, R18n.t.fronde.tasks.org.downloading)
     end
-  rescue RuntimeError
+  rescue RuntimeError, Interrupt
     warn R18n.t.fronde.tasks.org.no_download if verbose
   end
 
   desc 'Compile Org'
   multitask compile: ['var/tmp/org.tar.gz', 'lib'] do |task|
-    begin
-      # No need to force fetch last version as it is only interesting as
-      # part of the upgrade task
-      org_version = Fronde::Org.last_version
-    rescue RuntimeError
-      next
-    end
+    # No need to force fetch last version as it is only interesting as
+    # part of the upgrade task
+    org_version = Fronde::Org.last_version
+
     org_dir = "lib/org-#{org_version}"
     next if Dir.exist?("#{org_dir}/lisp")
 
@@ -54,6 +50,8 @@ namespace :org do
     else
       Fronde::CLI::Throbber.run(build, R18n.t.fronde.tasks.org.installing)
     end
+  rescue RuntimeError, Interrupt
+    next
   end
 
   directory 'lib'
@@ -76,10 +74,6 @@ namespace :org do
     Fronde::CONFIG.write_org_lisp_config
   end
 
-  file '.dir-locals.el' => 'var/lib/org-config.el' do
-    Fronde::Config::Helpers.write_dir_locals
-  end
-
   file '.gitignore' do
     next if File.exist? '.gitignore'
 
@@ -91,9 +85,10 @@ namespace :org do
 
   desc 'Install Org'
   multitask install: ['org:compile', '.gitignore'] do
-    # I need a fully installed org mode to correctly generate the lisp
-    # config
-    Rake::Task['.dir-locals.el'].invoke
+    # lib/htmlize.el and lib/ox-gmi.el cannot be generated in parallel
+    # of org:compilation, as it will leads to a weird SSL error. Thus
+    # finishing file generation "manually" here.
+    Rake::Task['var/lib/org-config.el'].invoke
     sources = Fronde::CONFIG.sources
     sources.each { mkdir_p _1['path'] }
 

data/lib/tasks/site.rake

@@ -10,7 +10,7 @@ namespace :site do
   task :build, [:force?] => ['var/lib/org-config.el'] do |_, args|
     args.with_defaults(force?: false)
     build_index = Thread.new do
-      all_index = Fronde::Index.all_html_blog_index
+      all_index = Fronde::Index.all_blog_index
       all_index.each do |index|
         index.write_all_org(verbose: verbose)
       end
@@ -25,19 +25,11 @@ namespace :site do
     end
     all_indexes = build_index[:all_indexes]
 
-    begin
-      build_html = Thread.new do
-        rm_r 'var/tmp/timestamps', force: true if args[:force?]
-        Fronde::Emacs.new(verbose: verbose).publish
-      end
-      Fronde::CLI::Throbber.run(build_html, R18n.t.fronde.tasks.site.building)
-
-    # :nocov:
-    rescue RuntimeError
-      warn R18n.t.fronde.tasks.site.aborting
-      next
+    build_html = Thread.new do
+      rm_r 'var/tmp/timestamps', force: true if args[:force?]
+      Fronde::Emacs.new(verbose: verbose).publish
     end
-    # :nocov:
+    Fronde::CLI::Throbber.run(build_html, R18n.t.fronde.tasks.site.building)
 
     if all_indexes.any?
       if verbose
@@ -65,6 +57,11 @@ namespace :site do
     Fronde::CLI::Throbber.run(
       customize_html, R18n.t.fronde.tasks.site.customizing
     )
+    # :nocov:
+  rescue RuntimeError, Interrupt
+    warn R18n.t.fronde.tasks.site.aborting
+    next
+    # :nocov:
   end
 
   desc 'Cleanup orphaned published files'

data/lib/tasks/sync.rake

@@ -1,40 +1,9 @@
 # frozen_string_literal: true
 
 require_relative '../fronde/config'
+require_relative '../fronde/sync'
 require_relative '../fronde/cli/throbber'
 
-module Fronde
-  class SyncError < ::StandardError; end
-end
-
-def rsync_command(test = nil)
-  rsync_command = Fronde::CONFIG.get('rsync')
-  return rsync_command unless rsync_command.nil?
-
-  optstring = []
-  optstring << 'n' if test
-  if verbose
-    optstring << 'v'
-  else
-    optstring << 'q'
-  end
-  "rsync -#{optstring.join}rlt --delete"
-end
-
-def pull_or_push(direction, type, test)
-  remote_path = Fronde::CONFIG.get("#{type}_remote")
-  raise Fronde::SyncError, "No #{type} remote path set" if remote_path.nil?
-
-  public_folder = Fronde::CONFIG.get("#{type}_public_folder")
-  # Default is to push
-  cmd = ["#{public_folder}/", remote_path]
-  cmd.reverse! if direction == :pull
-  rsync = rsync_command(test)
-  Thread.new do
-    sh "#{rsync} #{cmd.join(' ')}"
-  end
-end
-
 def source_types
   Fronde::CONFIG.sources.map(&:type).uniq
 end
@@ -43,7 +12,9 @@ namespace :sync do
   desc 'Push changes to server'
   task :push, :test? do |_, args|
     source_types.each do |type|
-      publish_thread = pull_or_push(:push, type, args[:test?])
+      publish_thread = Fronde::Sync.pull_or_push(
+        :push, type, test: args[:test?], verbose: verbose
+      )
       if verbose
         publish_thread.join
       else
@@ -51,16 +22,20 @@ namespace :sync do
           publish_thread, format('Publishing %<fmt>s:', fmt: type)
         )
       end
-    rescue Fronde::SyncError => e
+    rescue Fronde::Sync::Error => e
       warn e
       next
     end
+  rescue RuntimeError, Interrupt
+    next
   end
 
   desc 'Pull changes from server'
   task :pull, :test? do |_, args|
     source_types.each do |type|
-      pull_thread = pull_or_push(:pull, type, args[:test?])
+      pull_thread = Fronde::Sync.pull_or_push(
+        :pull, type, test: args[:test?], verbose: verbose
+      )
       if verbose
         pull_thread.join
       else
@@ -68,9 +43,11 @@ namespace :sync do
           pull_thread, format('Pulling %<fmt>s:', fmt: type)
         )
       end
-    rescue Fronde::SyncError => e
+    rescue Fronde::Sync::Error => e
       warn e
       next
     end
+  rescue RuntimeError, Interrupt
+    next
   end
 end

data/lib/tasks/tags.rake

@@ -5,7 +5,7 @@ require_relative '../fronde/index'
 namespace :tags do
   desc 'List all tags by name'
   task :name do
-    Fronde::Index.all_html_blog_index do |index|
+    Fronde::Index.all_blog_index do |index|
       next if index.empty?
 
       puts index.sort_by(:name).join("\n")
@@ -14,7 +14,7 @@ namespace :tags do
 
   desc 'List all tags by weight'
   task :weight do
-    Fronde::Index.all_html_blog_index do |index|
+    Fronde::Index.all_blog_index do |index|
       next if index.empty?
 
       puts index.sort_by(:weight).join("\n")

data/locales/en.yml

@@ -3,6 +3,7 @@ fronde:
   bin:
     usage: 'Usage: fronde %1 [options]'
     done: done
+    interrupted: interrupted
     commands:
       cmd_title: Commands
       alias: Alias for ‘%1’.

data/locales/fr.yml

@@ -3,6 +3,7 @@ fronde:
   bin:
     usage: 'Usage : fronde %1 [options]'
     done: fait
+    interrupted: arrêté
     commands:
       cmd_title: Commandes
       alias: Alias pour ‘%1’.