kozeki 0.1.0

data/lib/kozeki/dsl.rb

@@ -0,0 +1,65 @@
+module Kozeki
+  class Dsl
+    def initialize(options)
+      @options = options
+    end
+
+    attr_reader :options
+
+    def self.load_file(path, options: {})
+      d = self.new(options)
+      d.base_directory(File.dirname(path))
+      d.instance_eval(File.read(path), path, 1)
+      d
+    end
+
+    def self.eval(options: {}, &block)
+      d = self.new(options)
+      d.base_directory('.')
+      d.instance_eval(&block)
+      d
+    end
+
+    def base_directory(path)
+      @options[:base_directory] = path
+    end
+
+    def source_directory(path)
+      @options[:source_directory] = path
+    end
+
+    def destination_directory(path)
+      @options[:destination_directory] = path
+    end
+
+    def cache_directory(path)
+      @options[:cache_directory] = path
+    end
+
+    def collection_list_included_prefix(*prefixes)
+      (@options[:collection_list_included_prefix] ||= []).concat prefixes.flatten.map(&:to_s)
+    end
+
+    def collection_options(prefix:, **options)
+      @options[:collection_options] ||= []
+      # FIXME: recursive dependency :<
+      @options[:collection_options].push(Config::CollectionOptions.new(prefix:, **options))
+    end
+
+    def metadata_decorator(&block)
+      (@options[:metadata_decorators] ||= []).push(block)
+    end
+
+    def source_filesystem(x)
+      @options[:source_filesystem] = x
+    end
+
+    def destination_filesystem(x)
+      @options[:destination_filesystem] = x
+    end
+
+    def on_after_build(&block)
+      (@options[:after_build_callbacks] ||= []).push(block)
+    end
+  end
+end

data/lib/kozeki/filesystem.rb

@@ -0,0 +1,46 @@
+module Kozeki
+  module Filesystem
+    Entry = Data.define(:path, :mtime)
+    Event = Data.define(:op, :path, :time)
+
+    class NotFound < StandardError; end
+
+    def read(path)
+      read_with_mtime(path)[0]
+    end
+
+    def read_with_mtime(path)
+      raise NotImplementedError
+    end
+
+    def write(path, string)
+      raise NotImplementedError
+    end
+
+    def delete(path)
+      raise NotImplementedError
+    end
+
+    def list
+      list_entries.map(&:path)
+    end
+
+    def list_entries
+      raise NotImplementedError
+    end
+
+    def watch
+      raise NotImplementedError
+    end
+
+    def retain_only(files)
+      to_remove = list() - files
+      to_remove.each do |path|
+        delete(path)
+      end
+    end
+
+    def flush
+    end
+  end
+end

data/lib/kozeki/item.rb

@@ -0,0 +1,44 @@
+require 'time'
+
+module Kozeki
+  # Item represents a built item from Source.
+  class Item
+    class ParseError < StandardError; end
+
+    def self.load_from_json(json_string)
+      j = JSON.parse(json_string, symbolize_names: true)
+      raise ParseError, ".kind must be `item`" unless j[:kind] == 'item'
+      id = j.fetch(:id)
+      data = j.fetch(:data)
+      meta = j.fetch(:meta, {})
+      build = j.fetch(:kozeki_build, {})
+      Item.new(id:, data:, meta:, build:)
+    end
+
+    def initialize(id:, data:, meta: nil, build: nil)
+      @id = id
+      @data = data
+      @meta = meta
+      @build = build
+    end
+
+    attr_reader :id, :data, :meta, :build
+
+    def as_json
+      {
+        kind: 'item',
+        id: id,
+        meta: meta.transform_values do |v|
+          case v
+          when Time
+            v.xmlschema
+          else
+            v
+          end
+        end,
+        data:,
+        kozeki_build: build,
+      }
+    end
+  end
+end

data/lib/kozeki/loader_chain.rb

@@ -0,0 +1,20 @@
+module Kozeki
+  class LoaderChain
+    def initialize(loaders:, decorators:)
+      @loaders = loaders
+      @decorators = decorators
+    end
+
+    def try_read(...)
+      @loaders.each do |loader|
+        source = loader.try_read(...)
+        next unless source
+        @decorators.each do |decorator|
+          decorator.call(source.meta, source)
+        end
+        return source
+      end
+      nil
+    end
+  end
+end

data/lib/kozeki/local_filesystem.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+require 'kozeki/filesystem'
+require 'fileutils'
+
+module Kozeki
+  class LocalFilesystem
+    include Filesystem
+
+    def initialize(base_directory)
+      @base = base_directory
+    end
+
+    # @param path [Array]
+    def read(path)
+      File.read(File.join(@base, *path))
+    rescue Errno::ENOENT
+      raise Filesystem::NotFound
+    end
+
+    # @param path [Array]
+    def read_with_mtime(path)
+      [
+        read(path),
+        File.mtime(File.join(@base, *path)),
+      ]
+    end
+
+    # @param path [Array]
+    def write(path, string)
+      path = File.join(@base, *path)
+      dirname = File.dirname(path)
+      FileUtils.mkdir_p(dirname)
+      File.write(path, string)
+    end
+
+    # @param path [Array]
+    def delete(path)
+      File.unlink(File.join(@base, *path))
+    rescue Errno::ENOENT
+    end
+
+    def list_entries
+      range = File.join(@base, 'x')[0..-2].size .. -1
+      Dir[File.join(@base, '**', '*')].filter_map do |fspath|
+        path =  fspath[range].split(File::SEPARATOR)
+        next if File.directory?(fspath) rescue nil
+        Filesystem::Entry.new(
+          path:,
+          mtime: File.mtime(fspath),
+        )
+      end
+    end
+
+    def watch(&block)
+      require 'listen'
+      base = File.expand_path(@base)
+      l = Listen.to(@base) do |modified, added, removed|
+        yield [
+          *(modified + added).map do |path|
+            Filesystem::Event.new(
+              op: :update,
+              path: convert_absolute_to_path(base, path),
+              time: nil,
+            )
+          end,
+          *removed.map do |path|
+            Filesystem::Event.new(
+              op: :delete,
+              path: convert_absolute_to_path(base, path),
+              time: nil,
+            )
+          end,
+        ]
+      end
+      l.start
+      -> { l.stop }
+    end
+
+    private def convert_absolute_to_path(base, path)
+      Pathname.new(path).relative_path_from(base).to_s.split(File::SEPARATOR)
+    end
+  end
+end

data/lib/kozeki/markdown_loader.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+require 'yaml'
+require 'commonmarker'
+require 'kozeki/source'
+
+module Kozeki
+  module MarkdownLoader
+    def self.try_read(path:, filesystem:)
+      basename = path.last
+      return nil unless basename.end_with?('.md') || basename.end_with?('.mkd') || basename.end_with?('.markdown')
+      content, mtime = filesystem.read_with_mtime(path)
+      m = content.match(/\A\s*^---$(.+?)^---$/m)
+      meta = if m
+        YAML.safe_load(m[1], symbolize_names: true)
+      else
+        {}
+      end
+      Source.new(
+        path:,
+        meta:,
+        mtime:,
+        content:,
+        loader: self,
+      )
+    end
+
+    def self.build(source)
+      html = Commonmarker.to_html(source.content, options: {
+        render: {
+          unsafe: true,
+        },
+        extension: {
+          strikethrough: true,
+          tagfilter: false,
+          table: true,
+          autolink: true,
+          tasklist: true,
+          superscript: true,
+          header_ids: "#{source.id}--",
+          footnotes: true,
+          description_lists: true,
+          front_matter_delimiter: '---',
+          shortcodes: true,
+        },
+      })
+
+      {
+        html: html,
+      }
+    end
+  end
+end

data/lib/kozeki/queued_filesystem.rb

@@ -0,0 +1,65 @@
+require 'kozeki/filesystem'
+require 'thread'
+
+module Kozeki
+  class QueuedFilesystem
+    include Kozeki::Filesystem
+    Operation = Data.define(:method, :args)
+
+    def initialize(filesystem:, threads: 6)
+      @backend = filesystem
+      @lock = Mutex.new
+      @threads_num = threads
+      @threads = nil
+      @queue = nil
+      start
+    end
+
+    def start
+      @lock.synchronize do
+        queue = @queue = Queue.new
+        @threads = @threads_num.times.map { Thread.new(queue, &method(:do_worker)) }
+      end
+    end
+
+    def flush
+      return unless @threads
+      ths, q = @lock.synchronize do
+        [@threads, @queue]
+      end
+      start
+      q.close
+      ths.each(&:value)
+    end
+
+    def write(path, string)
+      @queue.push Operation.new(method: :write, args: [path, string])
+      nil
+    end
+
+    def delete(path)
+      @queue.push Operation.new(method: :delete, args: [path])
+      nil
+    end
+
+    private def do_worker(q)
+      while op = q.pop
+        case op.method
+        when :write
+          @backend.write(*op.args)
+        when :delete
+          @backend.delete(*op.args)
+        else
+          raise "unknown operation #{op.inspect}"
+        end
+      end
+    end
+
+    def read(...) = @backend.read(...)
+    def read_with_mtime(...) = @backend.read_with_mtime(...)
+    def list(...) = @backend.list(...)
+    def list_entries(...) = @backend.list_entries(...)
+    def watch(...) = @backend.watch(...)
+    def retain_only(...) = @backend.retain_only(...)
+  end
+end

data/lib/kozeki/record.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Kozeki
+  # Represents cached metadata of Item and Source in a State
+  Record = Data.define(:id, :path, :timestamp, :mtime, :meta, :build, :pending_build_action, :id_was) do
+    def self.from_source(s)
+      new(
+        path: s.path,
+        id: s.id,
+        timestamp: s.timestamp,
+        mtime: s.mtime,
+        meta: s.meta,
+        build: nil,
+        pending_build_action: nil,
+        id_was: nil,
+      )
+    end
+
+    def self.from_row(h)
+      new(
+        path: h.fetch('path').split('/'),
+        id: h.fetch('id'),
+        timestamp: Time.at(h.fetch('timestamp')),
+        mtime: Time.at(h.fetch('mtime')/1000.0),
+        meta: JSON.parse(h.fetch('meta'), symbolize_names: true),
+        build: h['build']&.then { JSON.parse(_1, symbolize_names: true) },
+        pending_build_action: h.fetch('pending_build_action', 'none')&.to_sym&.then { _1 == :none ? nil : _1 },
+        id_was: h.fetch('id_was', nil),
+      )
+    end
+
+    def path_row
+      path.join('/')
+    end
+
+    def to_row
+      {
+        path: path_row,
+        id:,
+        timestamp: timestamp.to_i,
+        mtime: (mtime.floor(4).to_f * 1000).truncate,
+        meta: JSON.generate(meta),
+        build: build && JSON.generate(build),
+        pending_build_action: pending_build_action&.to_s || 'none',
+      }
+    end
+  end
+end

data/lib/kozeki/source.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+require 'digest/sha2'
+
+require 'kozeki/record'
+require 'kozeki/item'
+
+module Kozeki
+  # Source represents a source text file.
+  class Source
+    # @param path [Array<String>]
+    def initialize(path:, meta:, mtime:, content:, loader:)
+      raise ArgumentError, "path fragment cannot include /" if path.any? { _1.include?('/') }
+      @path = path
+      @meta = meta
+      @mtime = mtime
+      @content = content
+
+      @loader = loader
+
+      raise ArgumentError, "path fragment cannot include /" if path.any? { _1.include?('/') }
+      raise ArgumentError, "id cannot include /" if id.include?('/')
+    end
+
+    attr_reader :path, :mtime, :content, :loader
+    attr_accessor :meta
+
+    def id
+      meta.fetch(:id) do
+        "ao_#{Digest::SHA256.hexdigest(path.join('/'))}"
+      end.to_s
+    end
+
+    def timestamp
+      meta[:timestamp]&.then { Time.xmlschema(_1) }
+    end
+
+    def collections
+      meta[:collections] || []
+    end
+
+    # Relative file path of built Item.
+    def item_path
+      ['items', "#{id}.json"]
+    end
+
+    def to_record
+      Record.from_source(self)
+    end
+
+    def build_item
+      data = loader.build(self)
+      Item.new(
+        id:,
+        data:,
+        meta:,
+        build: {},
+      )
+    end
+  end
+end