archival 0.0.2 → 0.0.7

data/lib/archival/builder.rb

@@ -3,43 +3,74 @@
 require 'liquid'
 require 'tomlrb'
 require 'tags/layout'
+require 'redcarpet'
 
 Liquid::Template.error_mode = :strict
 Liquid::Template.register_tag('layout', Layout)
 
 module Archival
+  class DuplicateKeyError < StandardError
+  end
+
   class Builder
     attr_reader :page_templates
 
     def initialize(config, *_args)
-      @config = Config.new(config)
+      @config = config
+      @markdown = Redcarpet::Markdown.new(
+        Redcarpet::Render::HTML.new(prettify: true,
+                                    hard_wrap: true), no_intra_emphasis: true,
+                                                      fenced_code_blocks: true,
+                                                      autolink: true,
+                                                      strikethrough: true,
+                                                      underline: true
+      )
       refresh_config
     end
 
     def refresh_config
       @file_system = Liquid::LocalFileSystem.new(
-        @config.root, '%s.liquid'
+        File.join(@config.root, @config.pages_dir), '%s.liquid'
       )
       @variables = {}
       @object_types = {}
       @page_templates = {}
+      @dynamic_pages = Set.new
+      @dynamic_templates = {}
 
-      Liquid::Template.file_system = @file_system
+      Liquid::Template.file_system = Liquid::LocalFileSystem.new(
+        File.join(@config.root, @config.pages_dir), '_%s.liquid'
+      )
 
       objects_definition_file = File.join(@config.root,
                                           'objects.toml')
       if File.file? objects_definition_file
-        @object_types = read_toml(objects_definition_file)
+        @object_types = Tomlrb.load_file(objects_definition_file)
       end
 
-      update_pages
       update_objects
+      update_pages
+    end
+
+    def full_rebuild
+      Layout.reset_cache
+      refresh_config
     end
 
     def update_pages
       do_update_pages(File.join(@config.root, @config.pages_dir))
     end
 
+    def dynamic?(file)
+      @dynamic_pages.include? File.basename(file, '.liquid')
+    end
+
+    def template_for_page(template_file)
+      content = @file_system.read_template_file(template_file)
+      content += dev_mode_content if @config.dev_mode
+      Liquid::Template.parse(content)
+    end
+
     def do_update_pages(dir, prefix = nil)
       add_prefix = lambda { |entry|
         prefix ? File.join(prefix, entry) : entry
@@ -53,16 +84,13 @@ module Archival
                          add_prefix(entry))
           end
         elsif File.file? File.join(dir, entry)
-          if entry.end_with?('.liquid') && !(entry.start_with? '_')
-            page_name = File.basename(entry,
-                                      '.liquid')
-            template_file = File.join(
-              @config.pages_dir,
-              add_prefix.call(page_name)
-            )
-            content = @file_system.read_template_file(template_file)
-            @page_templates[add_prefix.call(page_name)] =
-              Liquid::Template.parse(content)
+          page_name = File.basename(entry, '.liquid')
+          template_file = add_prefix.call(page_name)
+          if dynamic? entry
+            @dynamic_templates[template_file] = template_for_page(template_file)
+          elsif entry.end_with?('.liquid') && !(entry.start_with? '_')
+            @page_templates[template_file] =
+              template_for_page(template_file)
           end
         end
       end
@@ -75,30 +103,53 @@ module Archival
 
     def do_update_objects(dir)
       objects = {}
-      @object_types.each do |name, _definition|
-        objects[name] = []
+      @object_types.each do |name, definition|
+        objects[name] = {}
         obj_dir = File.join(dir, name)
         if File.directory? obj_dir
           Dir.foreach(obj_dir) do |file|
             if file.end_with? '.toml'
-              object = read_toml(File.join(
-                                   obj_dir, file
-                                 ))
+              object = Tomlrb.load_file(File.join(
+                                          obj_dir, file
+                                        ))
               object[:name] =
                 File.basename(file, '.toml')
-              objects[name].push object
+              objects[name][object[:name]] = parse_object(object, definition)
+              if definition.key? 'template'
+                @dynamic_pages << definition['template']
+              end
             end
           end
         end
-        objects[name] = objects[name].sort do |a, b|
-          (a['order'] || a[:name]).to_s <=> (b['order'] || b[:name]).to_s
-        end
+        objects[name] = sort_objects(objects[name])
       end
       @variables['objects'] = objects
     end
 
-    def read_toml(file_path)
-      Tomlrb.load_file(file_path)
+    def sort_objects(objects)
+      # Sort by either 'order' key or object name, depending on what is
+      # available.
+      sorted_by_keys = objects.sort_by do |name, obj|
+        obj.key?('order') ? obj['order'].to_s : name
+      end
+      sorted_objects = Archival::TemplateArray.new
+      sorted_by_keys.each do |d|
+        raise DuplicateKeyError if sorted_objects.key?(d[0])
+
+        sorted_objects.push(d[1])
+        sorted_objects[d[0]] = d[1]
+      end
+      sorted_objects
+    end
+
+    def parse_object(object, definition)
+      definition.each do |name, type|
+        case type
+        when 'markdown'
+          object[name] = @markdown.render(object[name]) if object[name]
+        end
+      end
+      object
     end
 
     def set_var(name, value)
@@ -110,18 +161,48 @@ module Archival
       template.render(@variables)
     end
 
+    def render_dynamic(page, obj)
+      template = @dynamic_templates[page]
+      template.render(@variables.merge({ page => obj }))
+    end
+
     def write_all
       Dir.mkdir(@config.build_dir) unless File.exist? @config.build_dir
       @page_templates.each_key do |template|
         out_dir = File.join(@config.build_dir,
                             File.dirname(template))
         Dir.mkdir(out_dir) unless File.exist? out_dir
-        out_path = File.join(@config.build_dir,
+        out_path = File.join(out_dir,
                              "#{template}.html")
         File.open(out_path, 'w+') do |file|
           file.write(render(template))
         end
       end
+      @dynamic_pages.each do |page|
+        out_dir = File.join(@config.build_dir, page)
+        Dir.mkdir(out_dir) unless File.exist? out_dir
+        objects = @variables['objects'][page]
+        objects.each do |obj|
+          out_path = File.join(out_dir, "#{obj[:name]}.html")
+          obj['path'] = out_path
+          File.open(out_path, 'w+') do |file|
+            file.write(render_dynamic(page, obj))
+          end
+        end
+      end
+      return if @config.dev_mode
+
+      # in production, also copy all assets to the dist folder.
+      @config.assets_dirs.each do |asset_dir|
+        FileUtils.copy_entry File.join(@config.root, asset_dir),
+                             File.join(@config.build_dir, asset_dir)
+      end
+    end
+
+    private
+
+    def dev_mode_content
+      "<script src=\"http://localhost:#{@config.helper_port}/js/archival-helper.js\" type=\"application/javascript\"></script>" # rubocop:disable Layout/LineLength
     end
   end
 end

data/lib/archival/config.rb

@@ -1,16 +1,30 @@
 # frozen_string_literal: true
 
+require 'tomlrb'
+
 module Archival
   class Config
-    attr_reader :pages_dir, :objects_dir, :root, :build_dir
+    attr_reader :pages_dir, :objects_dir, :assets_dirs, :root, :build_dir,
+                :helper_port, :dev_mode
 
-    def initialize(config)
-      @pages_dir = config['pages'] || 'pages'
-      @objects_dir = config['objects'] || 'objects'
+    def initialize(config = {})
       @root = config['root'] || Dir.pwd
-      @build_dir = config['build_dir'] || File.join(
+      manifest = load_manifest
+      @pages_dir = config['pages'] || manifest['pages'] || 'pages'
+      @objects_dir = config['objects'] || manifest['objects'] || 'objects'
+      @build_dir = config['build_dir'] || manifest['build_dir'] || File.join(
         @root, 'dist'
       )
+      @helper_port = config['helper_port'] || manifest['helper_port'] || 2701
+      @assets_dirs = config['assets_dirs'] || manifest['assets'] || []
+      @dev_mode = config[:dev_mode] || false
+    end
+
+    def load_manifest
+      manifest_file = File.join(@root, 'manifest.toml')
+      return Tomlrb.load_file(manifest_file) if File.file? manifest_file
+
+      {}
     end
   end
 end

data/lib/archival/helper_server.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require 'socket'
+require 'open-uri'
+
+module Archival
+  class HelperServer
+    attr_reader :page
+
+    def initialize(port, build_dir)
+      @port = port
+      @build_dir = build_dir
+      @helper_dir = File.expand_path(File.join(File.dirname(__FILE__),
+                                               '../../helper'))
+    end
+
+    def start
+      server = TCPServer.new @port
+      loop do
+        Thread.start(server.accept) do |client|
+          req = ''
+          method = nil
+          path = nil
+          while (line = client.gets) && (line != "\r\n")
+            unless method
+              req_info = line.split
+              method = req_info[0]
+              path = req_info[1]
+            end
+            req += line
+          end
+          client.close unless req
+          handle_request(client, req, method, path)
+        end
+      end
+    end
+
+    def refresh_client
+      ws_sendmessage('refresh')
+    end
+
+    private
+
+    MAGIC_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
+
+    def handle_request(client, req, method, path)
+      if method == 'GET' && path.start_with?('/js/')
+        # For static paths, just serve the files they refer to.
+        http_response(client, type: 'application/javascript') do
+          serve_static(client, path)
+        end
+        client.close
+      elsif (matches = req.match(/^Sec-WebSocket-Key: (\S+)/))
+        websocket_key = matches[1]
+        # puts "Websocket handshake detected with key: #{websocket_key}"
+        connect_socket(client, websocket_key)
+      else
+        client.close
+      end
+    end
+
+    def connect_socket(client, websocket_key)
+      @socket = client
+      response_key = Digest::SHA1.base64digest([websocket_key,
+                                                MAGIC_GUID].join)
+      #   puts "Responding to handshake with key: #{response_key}"
+
+      @socket.write "HTTP/1.1 101 Switching Protocols\r\n"
+      @socket.write "Upgrade: websocket\r\n"
+      @socket.write "Connection: Upgrade\r\n"
+      @socket.write "Sec-WebSocket-Accept: #{response_key}\r\n"
+      @socket.write "\r\n"
+
+      #   puts 'Handshake completed.'
+      ws_loop
+    end
+
+    def ws_loop
+      loop do
+        msg = ws_getmessage
+        next unless msg
+
+        if msg == 'connected'
+          ws_sendmessage('ready')
+        elsif msg.start_with?('page:')
+          page_path = Pathname.new(msg.sub(/^page:/, ''))
+          @page = page_path.relative_path_from(@build_dir)
+          ws_sendmessage('ok')
+        end
+      end
+    end
+
+    def validate_ws_message
+      first_byte = @socket.getbyte
+      return unless first_byte
+
+      fin = first_byte & 0b10000000
+      opcode = first_byte & 0b00001111
+
+      # Our server only supports single-frame, text messages.
+      # Raise an exception if the client tries to send anything else.
+      raise "We don't support continuations" unless fin
+      raise 'We only support opcode 1' unless opcode == 1
+
+      second_byte = @socket.getbyte
+      is_masked = second_byte & 0b10000000
+      payload_size = second_byte & 0b01111111
+
+      raise 'frame masked incorrectly' unless is_masked
+      raise 'payload must be < 126 bytes in length' unless payload_size < 126
+
+      payload_size
+    end
+
+    def ws_getmessage
+      payload_size = validate_ws_message
+      return unless payload_size
+
+      #   warn "Payload size: #{payload_size} bytes"
+
+      mask = 4.times.map { @socket.getbyte }
+      #   warn "Got mask: #{mask.inspect}"
+
+      data = payload_size.times.map { @socket.getbyte }
+      #   warn "Got masked data: #{data.inspect}"
+
+      unmasked_data = data.each_with_index.map do |byte, i|
+        byte ^ mask[i % 4]
+      end
+      #   warn "Unmasked the data: #{unmasked_data.inspect}"
+
+      unmasked_data.pack('C*').force_encoding('utf-8')
+    end
+
+    def ws_sendmessage(message)
+      return unless @socket
+
+      output = [0b10000001, message.size, message]
+      @socket.write output.pack("CCA#{message.size}")
+    end
+
+    def serve_static(client, path)
+      buffer = File.open(File.join(@helper_dir, path)).read
+      buffer.sub! '$PORT', @port.to_s
+      client.print buffer
+    end
+
+    def http_response(client, config)
+      status = config[:status] ||= 200
+      type = config[:type] ||= 'text/html'
+      client.print "HTTP/1.1 #{status}\r\n"
+      client.print "Content-Type: #{type}\r\n"
+      client.print "\r\n"
+      yield
+    end
+  end
+end

data/lib/archival/listen.rb

@@ -4,40 +4,83 @@ require 'listen'
 require 'pathname'
 
 module Archival
-  def self.child?(parent, child)
-    path = Pathname.new(child)
-    return true if path.fnmatch?(File.join(parent, '**'))
-
-    false
+  def listen(config = {})
+    @config = Config.new(config.merge(dev_mode: true))
+    builder = Builder.new(@config)
+    Logger.benchmark('built') do
+      builder.write_all
+    end
+    ignore = %r{/dist/}
+    listener = Listen.to(@config.root,
+                         ignore: ignore) do |modified, added, removed|
+      updated_pages = []
+      updated_objects = []
+      updated_assets = []
+      (modified + added + removed).each do |file|
+        case change_type(file)
+        when :pages
+          updated_pages << file
+        when :objects
+          updated_objects << file
+        when :assets
+          updated_assets << file
+        end
+      end
+      @server.refresh_client if rebuild?(builder, updated_objects,
+                                         updated_pages, updated_assets)
+    end
+    listener.start
+    serve_helpers
   end
 
-  def self.process_change?(file, builder)
-    if child?(File.join(@config.root, @config.pages_dir), file)
+  module_function :listen
+
+  class << self
+    private
+
+    def child?(parent, child)
+      path = Pathname.new(child)
+      return true if path.fnmatch?(File.join(parent, '**'))
+
+      false
+    end
+
+    def change_type(file)
       # a page was modified, rebuild the pages.
-      builder.update_pages
-      return true
-    elsif child?(File.join(@config.root, @config.objects_dir), file)
+      return :pages if child?(File.join(@config.root, @config.pages_dir),
+                              file)
       # an object was modified, rebuild the objects.
-      builder.update_objects
-      return true
+      return :objects if child?(File.join(@config.root, @config.objects_dir),
+                                file)
+
+      # layout and other assets. For now, this is everything.
+      @config.assets_dirs.each do |dir|
+        return :assets if child?(File.join(@config.root, dir), file)
+      end
+      return :assets if child?(File.join(@config.root, 'layout'), file)
+      return :assets if ['manifest.toml',
+                         'objects.toml'].include? File.basename(file)
+
+      :none
     end
-    false
-  end
 
-  def listen(config)
-    @config = Config.new(config)
-    builder = Builder.new(config)
-    builder.write_all
-    listener = Listen.to(@config.root) do |modified, added, removed|
-      needs_update = false
-      (modified + added + removed).each do |file|
-        needs_update = true if process_change?(file, builder)
+    def rebuild?(builder, updated_objects, updated_pages, updated_assets)
+      if updated_pages.empty? && updated_objects.empty? && updated_assets.empty?
+        return false
+      end
+
+      Logger.benchmark('rebuilt') do
+        builder.update_objects if updated_objects.length
+        builder.update_pages if updated_pages.length
+        builder.full_rebuild if updated_assets.length
+        builder.write_all
       end
-      builder.write_all if needs_update
+      true
     end
-    listener.start
-    listener
-  end
 
-  module_function :listen
+    def serve_helpers
+      @server = HelperServer.new(@config.helper_port, @config.build_dir)
+      @server.start
+    end
+  end
 end

data/lib/archival/logger.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+module Archival
+  class Logger
+    def self.benchmark(message, &block)
+      Benchmark.bm do |bm|
+        bm.report(message, &block)
+      end
+    end
+  end
+end

data/lib/archival/rake_tasks.rb

@@ -27,19 +27,15 @@ class RakeTasks
     build_dir = Dir.pwd
 
     task 'build' do
-      builder = Archival::Builder.new('root' => build_dir)
-      builder.write_all
+      Archival::Logger.benchmark('built') do
+        config = Archival::Config.new('root' => build_dir)
+        builder = Archival::Builder.new(config)
+        builder.write_all
+      end
     end
 
     task 'run' do
-      Archival.listen(build_dir)
-      begin
-        sleep
-      rescue Interrupt
-        # Don't print a stack when a user interrupts, as this is the right way
-        # to stop the development server.
-        puts ''
-      end
+      Archival.listen('root' => build_dir)
     end
 
     RakeTasks.instance = self

data/lib/archival/template_array.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Archival
+  class TemplateArray < Array
+    alias subscript_access []
+    alias subscript_write []=
+
+    def initialize(*args)
+      super(*args)
+      @data = {}
+    end
+
+    def [](*args)
+      key = args[0]
+      return @data[key] if key.is_a? String
+      return @data[key] if key.is_a? Symbol
+
+      subscript_access(*args)
+    end
+
+    def []=(*args)
+      key = args[0]
+      if key.is_a?(String) || key.is_a?(Symbol)
+        @data[key] = args[1]
+        return
+      end
+      subscript_write(*args)
+    end
+
+    def key?(key)
+      @data.key?(key)
+    end
+  end
+end

data/lib/archival/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Archival
-  VERSION = '0.0.2'
+  VERSION = '0.0.7'
 end

data/lib/archival.rb

@@ -5,6 +5,9 @@ module Archival
 end
 
 require 'archival/version'
+require 'archival/template_array'
+require 'archival/logger'
 require 'archival/config'
+require 'archival/helper_server'
 require 'archival/builder'
 require 'archival/listen'

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "archival",
-  "version": "0.0.2",
+  "version": "0.0.7",
   "description": "An incredibly simple CMS for durable websites",
   "bin": "build.rb",
   "directories": {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: archival
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Jesse Ditson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-11-05 00:00:00.000000000 Z
+date: 2021-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.7.0
+- !ruby/object:Gem::Dependency
+  name: redcarpet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.5.1
 - !ruby/object:Gem::Dependency
   name: tomlrb
   requirement: !ruby/object:Gem::Requirement
@@ -76,17 +90,22 @@ files:
 - bin/ldiff
 - bin/listen
 - bin/rake
+- bin/redcarpet
 - bin/rspec
 - bin/rubocop
 - bin/ruby-parse
 - bin/ruby-rewrite
 - bin/setup
 - exe/archival
+- helper/js/archival-helper.js
 - lib/archival.rb
 - lib/archival/builder.rb
 - lib/archival/config.rb
+- lib/archival/helper_server.rb
 - lib/archival/listen.rb
+- lib/archival/logger.rb
 - lib/archival/rake_tasks.rb
+- lib/archival/template_array.rb
 - lib/archival/version.rb
 - lib/tags/layout.rb
 - package.json