archival 0.0.0 → 0.0.5

data/bin/ruby-rewrite

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'ruby-rewrite' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("parser", "ruby-rewrite")

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/exe/archival

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'archival'
+
+VALID_COMMANDS = %w[
+  build
+  run
+].freeze
+
+command = ARGV[0]
+
+unless !command || VALID_COMMANDS.include?(command)
+  raise StandardError,
+        "Invalid command #{command}"
+end
+
+build_dir = Dir.pwd
+
+case command
+when 'build'
+  Archival::Logger.benchmark('built') do
+    config = Archival::Config.new('root' => build_dir)
+    builder = Archival::Builder.new(config)
+    builder.write_all
+  end
+when 'run'
+  Archival.listen('root' => build_dir)
+else
+  # print help
+  puts 'archival [command]'
+  puts ''
+  puts 'Commands:'
+  puts '      build     Builds the current directory as an archival website.'
+  puts '      run       Runs the current directory in development mode, '
+  + 'recompiling when files change.'
+end

data/helper/js/archival-helper.js

@@ -0,0 +1,91 @@
+/**
+ * When running archival in development mode (archival run), this file is
+ * injected into all pages, and is responsible for reloading the page when the
+ * source has changed.
+ */
+
+(function () {
+  const remotePort = $PORT;
+  const CONNECTING_COLOR = "#bd270d";
+  const CONNECTED_COLOR = "#19bd0d";
+  const CHECK_INTERVAL = 500;
+  const DISCONNECTED_INTERVAL = 1000;
+  const connectionDot = document.createElement("div");
+  connectionDot.style = `position: absolute; z-index: 9999; bottom: 10px; right: 10px; background-color: ${CONNECTING_COLOR}; width: 15px; height: 15px; border-radius: 50%; opacity: 0.8;`;
+  connectionDot.setAttribute("title", "Archival Dev Server: Connecting");
+  connectionDot.addEventListener(
+    "mouseenter",
+    () => (connectionDot.style.opacity = 0.2)
+  );
+  connectionDot.addEventListener(
+    "mouseleave",
+    () => (connectionDot.style.opacity = 0.8)
+  );
+
+  let lastContact = -1;
+  let isConnecting = false;
+  let connection;
+
+  function connectionLoop() {
+    connection.send(`page:${window.location.pathname}`);
+    if (Date.now() - lastContact > DISCONNECTED_INTERVAL) {
+      setConnected(false);
+      connectSocket();
+    }
+    setTimeout(connectionLoop, CHECK_INTERVAL);
+  }
+
+  function setConnected(connected) {
+    connectionDot.style.backgroundColor = connected
+      ? CONNECTED_COLOR
+      : CONNECTING_COLOR;
+    connectionDot.setAttribute(
+      "title",
+      `Archival Dev Server: ${connected ? "Connected" : "Disconnected"}`
+    );
+  }
+
+  window.onload = () => {
+    connectSocket(true);
+  };
+
+  function connectSocket(init) {
+    if (isConnecting) {
+      return;
+    }
+    isConnecting = true;
+    console.log(
+      `${init ? "connecting" : "reconnecting"} to archival dev server...`
+    );
+    document.body.appendChild(connectionDot);
+    connection = new WebSocket(`ws://localhost:${remotePort}`);
+    connection.onerror = () => {
+      isConnecting = false;
+    };
+
+    connection.onopen = () => {
+      isConnecting = false;
+      connection.send("connected");
+      if (init) {
+        connectionLoop();
+      }
+    };
+    connection.onmessage = (event) => {
+      lastContact = Date.now();
+      switch (event.data) {
+        case "ready":
+          console.log("connected to archival dev server.");
+          break;
+        case "ok":
+          setConnected(true);
+          break;
+        case "refresh":
+          window.location.reload();
+          break;
+        default:
+          console.log(`receieved unexpected message ${event.data}`);
+          break;
+      }
+    };
+  }
+})();

data/lib/archival/builder.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require 'liquid'
+require 'tomlrb'
+require 'tags/layout'
+require 'redcarpet'
+
+Liquid::Template.error_mode = :strict
+Liquid::Template.register_tag('layout', Layout)
+
+module Archival
+  class Builder
+    attr_reader :page_templates
+
+    def initialize(config, *_args)
+      @config = config
+      @markdown = Redcarpet::Markdown.new(
+        Redcarpet::Render::HTML.new(prettify: true,
+                                    hard_wrap: true), no_intra_emphasis: true,
+                                                      fenced_code_blocks: true,
+                                                      autolink: true,
+                                                      strikethrough: true,
+                                                      underline: true
+      )
+      refresh_config
+    end
+
+    def refresh_config
+      @file_system = Liquid::LocalFileSystem.new(
+        File.join(@config.root, @config.pages_dir), '%s.liquid'
+      )
+      @variables = {}
+      @object_types = {}
+      @page_templates = {}
+
+      Liquid::Template.file_system = Liquid::LocalFileSystem.new(
+        File.join(@config.root, @config.pages_dir), '_%s.liquid'
+      )
+
+      objects_definition_file = File.join(@config.root,
+                                          'objects.toml')
+      if File.file? objects_definition_file
+        @object_types = Tomlrb.load_file(objects_definition_file)
+      end
+
+      update_pages
+      update_objects
+    end
+
+    def full_rebuild
+      Layout.reset_cache
+      refresh_config
+    end
+
+    def update_pages
+      do_update_pages(File.join(@config.root, @config.pages_dir))
+    end
+
+    def do_update_pages(dir, prefix = nil)
+      add_prefix = lambda { |entry|
+        prefix ? File.join(prefix, entry) : entry
+      }
+      Dir.foreach(dir) do |entry|
+        if File.directory? entry
+          unless [
+            '.', '..'
+          ].include?(entry)
+            update_pages(File.join(dir, entry),
+                         add_prefix(entry))
+          end
+        elsif File.file? File.join(dir, entry)
+          if entry.end_with?('.liquid') && !(entry.start_with? '_')
+            page_name = File.basename(entry,
+                                      '.liquid')
+            template_file = add_prefix.call(page_name)
+            content = @file_system.read_template_file(template_file)
+            content += dev_mode_content if @config.dev_mode
+            @page_templates[add_prefix.call(page_name)] =
+              Liquid::Template.parse(content)
+          end
+        end
+      end
+    end
+
+    def update_objects
+      do_update_objects(File.join(@config.root,
+                                  @config.objects_dir))
+    end
+
+    def do_update_objects(dir)
+      objects = {}
+      @object_types.each do |name, definition|
+        objects[name] = {}
+        obj_dir = File.join(dir, name)
+        if File.directory? obj_dir
+          Dir.foreach(obj_dir) do |file|
+            if file.end_with? '.toml'
+              object = Tomlrb.load_file(File.join(
+                                          obj_dir, file
+                                        ))
+              object[:name] =
+                File.basename(file, '.toml')
+              objects[name][object[:name]] = parse_object(object, definition)
+            end
+          end
+        end
+        objects[name] = sort_objects(objects[name])
+      end
+      @variables['objects'] = objects
+    end
+
+    def sort_objects(objects)
+      # Since objects are hashes but we'd like them to be iterable based on
+      # arbitrary "order" keys, and in ruby hashes enumerate in insert order,
+      # we just need to re-insert in the correct order.
+      sorted_keys = objects.sort_by do |name, obj|
+        obj['order'].to_s || name
+      end
+      sorted_objects = {}
+      sorted_keys.each do |d|
+        sorted_objects[d[0]] = d[1]
+      end
+      sorted_objects
+    end
+
+    def parse_object(object, definition)
+      definition.each do |name, type|
+        case type
+        when 'markdown'
+          object[name] = @markdown.render(object[name]) if object[name]
+        end
+      end
+      object
+    end
+
+    def set_var(name, value)
+      @variables[name] = value
+    end
+
+    def render(page)
+      template = @page_templates[page]
+      template.render(@variables)
+    end
+
+    def write_all
+      Dir.mkdir(@config.build_dir) unless File.exist? @config.build_dir
+      @page_templates.each_key do |template|
+        out_dir = File.join(@config.build_dir,
+                            File.dirname(template))
+        Dir.mkdir(out_dir) unless File.exist? out_dir
+        out_path = File.join(@config.build_dir,
+                             "#{template}.html")
+        File.open(out_path, 'w+') do |file|
+          file.write(render(template))
+        end
+      end
+      return if @config.dev_mode
+
+      # in production, also copy all assets to the dist folder.
+      @config.assets_dirs.each do |asset_dir|
+        FileUtils.copy_entry File.join(@config.root, asset_dir),
+                             File.join(@config.build_dir, asset_dir)
+      end
+    end
+
+    private
+
+    def dev_mode_content
+      "<script src=\"http://localhost:#{@config.helper_port}/js/archival-helper.js\" type=\"application/javascript\"></script>" # rubocop:disable Layout/LineLength
+    end
+  end
+end

data/lib/archival/config.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require 'tomlrb'
+
+module Archival
+  class Config
+    attr_reader :pages_dir, :objects_dir, :assets_dirs, :root, :build_dir,
+                :helper_port, :dev_mode
+
+    def initialize(config = {})
+      @root = config['root'] || Dir.pwd
+      manifest = load_manifest
+      @pages_dir = config['pages'] || manifest['pages'] || 'pages'
+      @objects_dir = config['objects'] || manifest['objects'] || 'objects'
+      @build_dir = config['build_dir'] || manifest['build_dir'] || File.join(
+        @root, 'dist'
+      )
+      @helper_port = config['helper_port'] || manifest['helper_port'] || 2701
+      @assets_dirs = config['assets_dirs'] || manifest['assets'] || []
+      @dev_mode = config[:dev_mode] || false
+    end
+
+    def load_manifest
+      manifest_file = File.join(@root, 'manifest.toml')
+      return Tomlrb.load_file(manifest_file) if File.file? manifest_file
+
+      {}
+    end
+  end
+end

data/lib/archival/helper_server.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'open-uri'
+
+module Archival
+  class HelperServer
+    attr_reader :page
+
+    def initialize(port, build_dir)
+      @port = port
+      @build_dir = build_dir
+      @helper_dir = File.expand_path(File.join(File.dirname(__FILE__),
+                                               '../../helper'))
+    end
+
+    def start
+      server = TCPServer.new @port
+      loop do
+        Thread.start(server.accept) do |client|
+          req = ''
+          method = nil
+          path = nil
+          while (line = client.gets) && (line != "\r\n")
+            unless method
+              req_info = line.split
+              method = req_info[0]
+              path = req_info[1]
+            end
+            req += line
+          end
+          client.close unless req
+          handle_request(client, req, method, path)
+        end
+      end
+    end
+
+    def refresh_client
+      ws_sendmessage('refresh')
+    end
+
+    private
+
+    MAGIC_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
+
+    def handle_request(client, req, method, path)
+      if method == 'GET' && path.start_with?('/js/')
+        # For static paths, just serve the files they refer to.
+        http_response(client, type: 'application/javascript') do
+          serve_static(client, path)
+        end
+        client.close
+      elsif (matches = req.match(/^Sec-WebSocket-Key: (\S+)/))
+        websocket_key = matches[1]
+        # puts "Websocket handshake detected with key: #{websocket_key}"
+        connect_socket(client, websocket_key)
+      else
+        client.close
+      end
+    end
+
+    def connect_socket(client, websocket_key)
+      @socket = client
+      response_key = Digest::SHA1.base64digest([websocket_key,
+                                                MAGIC_GUID].join)
+      #   puts "Responding to handshake with key: #{response_key}"
+
+      @socket.write "HTTP/1.1 101 Switching Protocols\r\n"
+      @socket.write "Upgrade: websocket\r\n"
+      @socket.write "Connection: Upgrade\r\n"
+      @socket.write "Sec-WebSocket-Accept: #{response_key}\r\n"
+      @socket.write "\r\n"
+
+      #   puts 'Handshake completed.'
+      ws_loop
+    end
+
+    def ws_loop
+      loop do
+        msg = ws_getmessage
+        next unless msg
+
+        if msg == 'connected'
+          ws_sendmessage('ready')
+        elsif msg.start_with?('page:')
+          page_path = Pathname.new(msg.sub(/^page:/, ''))
+          @page = page_path.relative_path_from(@build_dir)
+          ws_sendmessage('ok')
+        end
+      end
+    end
+
+    def validate_ws_message
+      first_byte = @socket.getbyte
+      return unless first_byte
+
+      fin = first_byte & 0b10000000
+      opcode = first_byte & 0b00001111
+
+      # Our server only supports single-frame, text messages.
+      # Raise an exception if the client tries to send anything else.
+      raise "We don't support continuations" unless fin
+      raise 'We only support opcode 1' unless opcode == 1
+
+      second_byte = @socket.getbyte
+      is_masked = second_byte & 0b10000000
+      payload_size = second_byte & 0b01111111
+
+      raise 'frame masked incorrectly' unless is_masked
+      raise 'payload must be < 126 bytes in length' unless payload_size < 126
+
+      payload_size
+    end
+
+    def ws_getmessage
+      payload_size = validate_ws_message
+      return unless payload_size
+
+      #   warn "Payload size: #{payload_size} bytes"
+
+      mask = 4.times.map { @socket.getbyte }
+      #   warn "Got mask: #{mask.inspect}"
+
+      data = payload_size.times.map { @socket.getbyte }
+      #   warn "Got masked data: #{data.inspect}"
+
+      unmasked_data = data.each_with_index.map do |byte, i|
+        byte ^ mask[i % 4]
+      end
+      #   warn "Unmasked the data: #{unmasked_data.inspect}"
+
+      unmasked_data.pack('C*').force_encoding('utf-8')
+    end
+
+    def ws_sendmessage(message)
+      return unless @socket
+
+      output = [0b10000001, message.size, message]
+      @socket.write output.pack("CCA#{message.size}")
+    end
+
+    def serve_static(client, path)
+      buffer = File.open(File.join(@helper_dir, path)).read
+      buffer.sub! '$PORT', @port.to_s
+      client.print buffer
+    end
+
+    def http_response(client, config)
+      status = config[:status] ||= 200
+      type = config[:type] ||= 'text/html'
+      client.print "HTTP/1.1 #{status}\r\n"
+      client.print "Content-Type: #{type}\r\n"
+      client.print "\r\n"
+      yield
+    end
+  end
+end

data/lib/archival/listen.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'listen'
+require 'pathname'
+
+module Archival
+  def listen(config = {})
+    @config = Config.new(config.merge(dev_mode: true))
+    builder = Builder.new(@config)
+    Logger.benchmark('built') do
+      builder.write_all
+    end
+    ignore = %r{/dist/}
+    listener = Listen.to(@config.root,
+                         ignore: ignore) do |modified, added, removed|
+      updated_pages = []
+      updated_objects = []
+      updated_assets = []
+      (modified + added + removed).each do |file|
+        case change_type(file)
+        when :pages
+          updated_pages << file
+        when :objects
+          updated_objects << file
+        when :assets
+          updated_assets << file
+        end
+      end
+      @server.refresh_client if rebuild?(builder, updated_objects,
+                                         updated_pages, updated_assets)
+    end
+    listener.start
+    serve_helpers
+  end
+
+  module_function :listen
+
+  class << self
+    private
+
+    def child?(parent, child)
+      path = Pathname.new(child)
+      return true if path.fnmatch?(File.join(parent, '**'))
+
+      false
+    end
+
+    def change_type(file)
+      # a page was modified, rebuild the pages.
+      return :pages if child?(File.join(@config.root, @config.pages_dir),
+                              file)
+      # an object was modified, rebuild the objects.
+      return :objects if child?(File.join(@config.root, @config.objects_dir),
+                                file)
+
+      # layout and other assets. For now, this is everything.
+      @config.assets_dirs.each do |dir|
+        return :assets if child?(File.join(@config.root, dir), file)
+      end
+      return :assets if child?(File.join(@config.root, 'layout'), file)
+      return :assets if ['manifest.toml',
+                         'objects.toml'].include? File.basename(file)
+
+      :none
+    end
+
+    def rebuild?(builder, updated_objects, updated_pages, updated_assets)
+      if updated_pages.empty? && updated_objects.empty? && updated_assets.empty?
+        return false
+      end
+
+      Logger.benchmark('rebuilt') do
+        builder.update_objects if updated_objects.length
+        builder.update_pages if updated_pages.length
+        builder.full_rebuild if updated_assets.length
+        builder.write_all
+      end
+      true
+    end
+
+    def serve_helpers
+      @server = HelperServer.new(@config.helper_port, @config.build_dir)
+      @server.start
+    end
+  end
+end

data/lib/archival/logger.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+module Archival
+  class Logger
+    def self.benchmark(message, &block)
+      Benchmark.bm do |bm|
+        bm.report(message, &block)
+      end
+    end
+  end
+end

data/lib/archival/rake_tasks.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rake'
+require 'rake/tasklib'
+require 'archival'
+
+class RakeTasks
+  # Common tasks for archival.
+  #
+  # To include, just add
+  #
+  # require 'archival/rake_tasks'
+  #
+  # to your Rakefile.
+  include Rake::DSL if defined? Rake::DSL
+
+  class << self
+    # set when install'd.
+    attr_accessor :instance
+
+    def install_tasks
+      new.install
+    end
+  end
+
+  def install
+    build_dir = Dir.pwd
+
+    task 'build' do
+      Archival::Logger.benchmark('built') do
+        config = Archival::Config.new('root' => build_dir)
+        builder = Archival::Builder.new(config)
+        builder.write_all
+      end
+    end
+
+    task 'run' do
+      Archival.listen('root' => build_dir)
+    end
+
+    RakeTasks.instance = self
+  end
+end
+
+RakeTasks.install_tasks

data/lib/archival/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Archival
+  VERSION = '0.0.5'
+end

data/lib/archival.rb

@@ -1,3 +1,12 @@
+# frozen_string_literal: true
 
-class Archival
-end
+module Archival
+  # Main Archival module. See https://archival.dev for docs.
+end
+
+require 'archival/version'
+require 'archival/logger'
+require 'archival/config'
+require 'archival/helper_server'
+require 'archival/builder'
+require 'archival/listen'