kozeki 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 030de1696d26b4f3cf6ed9616dd273f1610cbc63dea404e0250950d587fcd581
+  data.tar.gz: 3a6c8652c52482442ed17dc5c057912c6792b65f64989954b9497e9c7a6aaa04
+SHA512:
+  metadata.gz: 4e177182d9a52715721706494b858183c42c4391b4e8ac37890173bc15bcf1f6ec674d16ae5b00634f3ecefc9d211f08e10cacbf62e4db4ac60804ca408fa6da
+  data.tar.gz: e288bd18e158851513a2264ba5573e7980dfa8afc618633b805e8177473c2f8d51e7656b6a9ac3cfb23d20b183a1cf70887db355a00b7ccae20dc582de375229

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-08-06
+
+- Initial release

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/kozeki

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'kozeki'
+
+Kozeki::Cli.start

data/lib/kozeki/build.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+require 'json'
+require 'kozeki/state'
+require 'kozeki/item'
+require 'kozeki/collection'
+require 'kozeki/collection_list'
+
+module Kozeki
+  class Build
+
+    # @param state [State]
+    def initialize(state:, source_filesystem:, destination_filesystem:, collection_list_included_prefix: nil, collection_options: [], loader:, events: nil, incremental_build:, logger: nil)
+      @state = state
+
+      @source_filesystem = source_filesystem
+      @destination_filesystem = destination_filesystem
+      @collection_list_included_prefix = collection_list_included_prefix
+      @collection_options = collection_options
+      @loader = loader
+      @loader_cache = {}
+      @logger = logger
+
+      @events = events
+      @incremental_build = incremental_build
+
+      @updated_files = []
+      @build_id = nil
+    end
+
+    attr_accessor :events, :incremental_build
+    attr_reader :updated_files
+
+    def inspect
+      "#<#{self.class.name}#{self.__id__}>"
+    end
+
+    def incremental_build_possible?
+      @state.build_exist?
+    end
+
+    def incremental_build?
+      incremental_build_possible? && @incremental_build
+    end
+
+    def full_build?
+      !incremental_build?
+    end
+
+    def perform
+      raise "can't reuse" if @build_id
+      if full_build?
+        @logger&.info "Starting full build"
+      else
+        @logger&.info "Starting incremental build"
+      end
+
+      @state.transaction do
+        process_prepare
+        process_events
+      end
+      @state.transaction do
+        process_items_remove
+        process_items_update
+      end
+      @state.transaction do
+        process_garbage
+      end
+      @state.transaction do
+        process_collections
+      end
+      @destination_filesystem.flush
+      @state.transaction do
+        process_commit
+      end
+      @destination_filesystem.flush
+    end
+
+    private def process_prepare
+      @logger&.debug "=== Prepare ==="
+      @state.clear! if full_build?
+      @build_id = @state.create_build
+      @logger&.debug "Build ID: #{@build_id.inspect}"
+    end
+
+    private def process_events
+      @logger&.debug "=== Process incoming events ==="
+      events = if full_build? || @events.nil?
+        fs_list = @source_filesystem.list_entries()
+        fs_list.map do |entry|
+          Filesystem::Event.new(op: :update, path: entry.path, time: full_build? ? nil : entry.mtime.floor(4))
+        end
+      else
+        @events.dup
+      end
+
+      if fs_list
+        seen_paths = {}
+        events.each do |event|
+          next unless event.op == :update
+          seen_paths[event.path] = true
+        end
+        @state.list_record_paths.reject do |path|
+          seen_paths[path]
+        end.each do |path|
+          events.push(Filesystem::Event.new(op: :delete, path:, time: nil))
+        end
+      end
+
+      events.each do |event|
+        @logger&.debug "> #{event.inspect}" if incremental_build? && @events
+        case event.op
+        when :update
+          if event.time
+            begin
+              record = @state.find_record_by_path!(event.path)
+              diff = event.time.to_f.floor(3) - record.mtime.to_f.floor(3)
+              if diff > 0.005
+                @logger&.debug "> #{event.inspect}"
+                @logger&.debug "  #{record.mtime} (#{record.mtime.to_f.floor(3)}) < #{event.time} (#{event.time.to_f.floor(3)})"
+              else
+                next
+              end
+            rescue State::NotFound
+            end
+          end
+
+          source = load_source(event.path)
+          record = @state.save_record(source.to_record)
+          @state.set_record_pending_build_action(record, :update)
+          @logger&.info "ID change: #{event.path.inspect}; #{record.id_was.inspect} => #{record.id.inspect}" if record.id_was
+        when :delete
+          begin
+            record = @state.find_record_by_path!(event.path)
+            @state.set_record_pending_build_action(record, :remove)
+          rescue State::NotFound
+          end
+        else
+          raise "unknown op #{event.inspect}"
+        end
+      end
+    end
+
+    private def process_items_remove
+      @logger&.debug "=== Delete items for removed sources ==="
+      removed_records = @state.list_records_by_pending_build_action(:remove)
+      removed_records.each do |record|
+        if @state.list_records_by_id(record.id).any? { _1.pending_build_action == :update }
+          @logger&.warn "Skip deletion: #{record.id.inspect} (#{record.path.inspect})"
+          next
+        end
+        @logger&.info "Delete: #{record.id.inspect} (#{record.path.inspect})"
+        @destination_filesystem.delete(['items', "#{record.id}.json"])
+        @state.set_record_collections_pending(record.id, [])
+      end
+    end
+
+    private def process_items_update
+      @logger&.debug "=== Render items for updated sources ==="
+      updating_records = @state.list_records_by_pending_build_action(:update)
+      updating_records.each do |record|
+        @logger&.info "Render: #{record.id.inspect} (#{record.path.inspect})"
+        source = load_source(record.path)
+
+        # ID uniqueness check
+        _existing_record = begin
+          @state.find_record!(source.id)
+        rescue State::NotFound; nil
+        end
+
+        item = build_item(source)
+        @destination_filesystem.write(source.item_path, "#{JSON.generate(item.as_json)}\n")
+        @state.set_record_collections_pending(item.id, item.meta.fetch(:collections, []))
+        @updated_files << source.item_path
+      end
+    end
+
+    private def process_garbage
+      @logger&.debug "=== Collect garbages; items without source ==="
+      item_ids = @state.list_item_ids_for_garbage_collection
+      item_ids.each do |item_id|
+        @logger&.debug "Checking: #{item_id.inspect}"
+        records = @state.list_records_by_id(item_id)
+        if records.empty?
+          @logger&.info "Garbage: #{item_id.inspect}"
+          @state.mark_item_id_to_remove(item_id)
+          @state.set_record_collections_pending(item_id, [])
+          @destination_filesystem.delete(['items', "#{item_id}.json"])
+        end
+      end
+    end
+
+    private def process_collections
+      @logger&.debug "=== Render updated collections ==="
+      collections = @state.list_collection_names_pending
+      return if collections.empty?
+
+      collections.each do |collection_name|
+        records = @state.list_collection_records(collection_name)
+        record_count_was = @state.count_collection_records(collection_name)
+
+        collection = make_collection(collection_name, records)
+        collection.pages.each do |page|
+          @logger&.info "Render: Collection #{collection_name.inspect} (#{page.item_path.inspect})"
+          @destination_filesystem.write(page.item_path, "#{JSON.generate(page.as_json)}\n")
+          @updated_files << page.item_path
+        end
+        collection.item_paths_for_missing_pages(record_count_was).each do |path|
+          @logger&.info "Delete: Collection #{collection.inspect} (#{path.inspect})"
+          @destination_filesystem.delete(path)
+        end
+      end
+
+      @logger&.info "Render: CollectionList"
+      collection_names = @state.list_collection_names_with_prefix(*@collection_list_included_prefix)
+      collection_list = CollectionList.new(collection_names)
+      @destination_filesystem.write(collection_list.item_path, "#{JSON.generate(collection_list.as_json)}\n")
+      @updated_files << collection_list.item_path
+    end
+
+    private def process_commit
+      @logger&.debug "=== Finishing build ==="
+      @logger&.debug "Flush pending actions"
+      @state.process_markers!
+      if full_build?
+        @logger&.info "Delete: untouched files from destination"
+        @destination_filesystem.retain_only(@updated_files)
+      end
+      @logger&.debug "Mark build completed"
+      @state.mark_build_completed(@build_id)
+      # TODO: @state.remove_old_builds
+    end
+
+    private def load_source(path)
+      @loader_cache[path] ||= begin
+        @logger&.debug("Load: #{path.inspect}")
+        @loader.try_read(path: path, filesystem: @source_filesystem) or raise "can't read #{path.inspect}"
+      end
+    end
+
+    private def build_item(source)
+      source.build_item
+    end
+
+    private def make_collection(name, records)
+      Collection.new(name, records, options: collection_option_for(name))
+    end
+
+    private def collection_option_for(collection_name)
+      retval = nil
+      len = -1
+      @collection_options.each do |x|
+        if collection_name.start_with?(x.prefix) && x.prefix.size > len
+          retval = x
+          len = x.prefix.size
+        end
+      end
+      retval
+    end
+  end
+end

data/lib/kozeki/cli.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+require 'thor'
+require 'json'
+
+require 'kozeki/client'
+require 'kozeki/config'
+require 'kozeki/filesystem'
+
+module Kozeki
+  class Cli < Thor
+    package_name 'kozeki'
+
+    desc 'build CONFIG_FILE', 'run a one-off build using CONFIG_FILE'
+    method_options :full => :boolean
+    method_options :events_from_stdin => :boolean
+    def build(config_file)
+      client = make_client(config_file)
+      client.build(
+        incremental_build: !options[:full],
+        events: options[:events_from_stdin] ? load_events_from_stdin() : nil,
+      )
+    end
+
+    desc 'watch CONFIG_FILE', 'run a continuous build by watching source filesystem using CONFIG_FILE'
+    def watch(config_file)
+      client = make_client(config_file)
+      client.watch
+    end
+
+    desc 'debug-state CONFIG_FILE', ''
+    def debug_state(config_file)
+      client = make_client(config_file)
+      state = State.open(path: client.config.state_path)
+      state.db.execute(%{select * from "records" order by "id" asc}).map { p record: _1 }
+      state.db.execute(%{select * from "collection_memberships" order by "collection" asc, "record_id" asc}).map { p collection_membership: _1 }
+      state.db.execute(%{select * from "item_ids" order by "id" asc}).map { p item_id: _1 }
+    end
+
+    no_commands do
+      private def make_client(config_file)
+        config = Config.load_file(config_file)
+        Client.new(config:)
+      end
+
+      private def load_events_from_stdin
+        j = JSON.parse($stdin.read, symbolize_names: true)
+        j.map do |x|
+          Filesystem::Event.new(
+            op: x.fetch(:op).to_sym,
+            path: x.fetch(:path),
+            time: nil,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/kozeki/client.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+require 'kozeki/state'
+require 'kozeki/build'
+
+module Kozeki
+  class Client
+    # @param config [Config]
+    def initialize(config:)
+      @config = config
+    end
+
+    attr_reader :config
+
+    def build(incremental_build: true, events: nil)
+      begin
+        state = State.open(path: config.state_path)
+        build = Build.new(
+          state: state,
+          source_filesystem: @config.source_filesystem,
+          destination_filesystem: @config.destination_filesystem,
+          collection_list_included_prefix: @config.collection_list_included_prefix,
+          collection_options: @config.collection_options,
+          loader: @config.loader,
+          incremental_build:,
+          events:,
+          logger: @config.logger,
+        )
+        build.perform
+      ensure
+        state&.close
+      end
+
+      @config.after_build_callbacks.each do |cb|
+        cb.call(build)
+      end
+    end
+
+    def watch
+      build(incremental_build: true, events: nil)
+      stop = @config.source_filesystem.watch do |events|
+        build(incremental_build: true, events: events)
+      end
+      sleep
+    ensure
+      stop&.call
+    end
+  end
+end

data/lib/kozeki/collection.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+require 'time'
+
+module Kozeki
+  class Collection
+    def initialize(name, records, options: nil)
+      raise ArgumentError, "name cannot include /" if name.include?('/')
+      @name = name
+      @records = records
+      @options = options
+
+      @records_sorted = nil
+    end
+
+    attr_reader :name, :records, :options
+
+    def records_sorted
+      @records_sorted ||= @records.sort_by do |record|
+        record.timestamp&.then { -_1.to_i } || 0
+      end
+    end
+
+    def total_pages
+      @total_pages ||= calculate_total_pages(records.size)
+    end
+
+    def pages
+      case
+      when records.empty?
+        []
+      when options&.paginate && options&.max_items
+        total_pages.times.map do |i|
+          Page.new(parent: self, page: i+1)
+        end
+      else
+        [Page.new(parent: self, page: nil)]
+      end
+    end
+
+    def item_path_for_page(pagenum)
+      case pagenum
+      when 0
+        raise "[bug] page is 1-origin"
+      when nil, 1
+        ['collections', "#{name}.json"]
+      else
+        ['collections', "#{name}", "page-#{pagenum}.json"]
+      end
+    end
+
+    def item_paths_for_missing_pages(item_count_was)
+      total_pages_was = calculate_total_pages(item_count_was)
+      if (total_pages_was - total_pages) > 0
+        (total_pages+1..total_pages_was).map do |pagenum|
+          item_path_for_page(pagenum)
+        end
+      else
+        []
+      end
+    end
+
+    def calculate_total_pages(count)
+      if options&.paginate && options&.max_items
+        count.divmod(options.max_items).then {|(a,b)| a + (b>0 ? 1 : 0) }
+      else
+        count > 0 ? 1 : 0
+      end
+    end
+
+    class Page
+      def initialize(parent:, page:)
+        @parent = parent
+        @page = page
+
+        @total_pages = nil
+      end
+
+      def name; @parent.name; end
+      def options; @parent.options; end
+      def total_pages; @parent.total_pages; end
+
+      def records
+        case @page
+        when nil
+          @parent.records_sorted
+        when 0
+          raise "[bug] page is 1-origin"
+        else
+          @parent.records_sorted[(@page - 1) * options.max_items, options.max_items]
+        end
+      end
+
+      def as_json
+        {
+          kind: 'collection',
+          name: name,
+          items: records.map do |record|
+            {
+              id: record.id,
+              path: ['items', "#{record.id}.json"].join('/'),
+              meta: options&.meta_keys ? record.meta.slice(*options.meta_keys) : record.meta,
+            }
+          end.then do |page|
+            options&.max_items ? page[0, options.max_items] : page
+          end,
+        }.tap do |j|
+          j[:page] = page_info if @page
+        end
+      end
+
+      def item_path
+        @parent.item_path_for_page(@page)
+      end
+
+      def page_info
+        return nil unless @page
+        prev_page = 1 < @page ? @parent.item_path_for_page(@page-1) : nil
+        next_page = @page < total_pages ? @parent.item_path_for_page(@page+1) : nil
+        i = {
+          self: @page,
+          total_pages:,
+          first: @parent.item_path_for_page(1).join('/'),
+          last: @parent.item_path_for_page(total_pages).join('/'),
+          prev: prev_page&.join('/'),
+          next: next_page&.join('/'),
+        }
+        if @page == 1
+          i[:pages] = (1..total_pages).map do |pagenum|
+            @parent.item_path_for_page(pagenum).join('/')
+          end
+        end
+        i
+      end
+    end
+  end
+end

data/lib/kozeki/collection_list.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kozeki
+  class CollectionList
+    def initialize(names)
+      @names = names
+    end
+
+    attr_reader :names
+
+    def as_json
+      {
+        kind: 'collection_list',
+        collections: names.sort.map do |name|
+          {
+            name:,
+            path: ['collections', "#{name}.json"].join('/'),
+          }
+        end,
+      }
+    end
+
+    def item_path
+      ['collections.json']
+    end
+  end
+end

data/lib/kozeki/config.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+require 'kozeki/dsl'
+require 'kozeki/loader_chain'
+require 'kozeki/local_filesystem'
+require 'kozeki/markdown_loader'
+require 'logger'
+
+module Kozeki
+  class Config
+    CollectionOptions = Struct.new(:prefix, :max_items, :paginate, :meta_keys)
+
+    def initialize(options)
+      @options = options
+      @source_filesystem = nil
+      @destination_filesystem = nil
+      @loader = nil
+      @logger = nil
+    end
+
+    def self.load_file(path)
+      new(Dsl.load_file(path).options)
+    end
+
+    def self.configure(&block)
+      new(Dsl.eval(&block).options)
+    end
+
+    def [](k)
+      @options[k]
+    end
+
+    def fetch(k,*args)
+      @options.fetch(k,*args)
+    end
+
+    def base_directory
+      @options.fetch(:base_directory, '.')
+    end
+
+    def source_directory
+      @options.fetch(:source_directory)
+    end
+
+    def destination_directory
+      @options.fetch(:destination_directory)
+    end
+
+    def cache_directory
+      @options.fetch(:cache_directory, nil)
+    end
+
+    def collection_list_included_prefix
+      @options.fetch(:collection_list_included_prefix, nil)
+    end
+
+    def collection_options
+      @options.fetch(:collection_options, [])
+    end
+
+    def metadata_decorators
+      @options.fetch(:metadata_decorators, [])
+    end
+
+    def after_build_callbacks
+      @options.fetch(:after_build_callbacks, [])
+    end
+
+    def state_path
+      cache_directory ? File.join(File.expand_path(cache_directory, base_directory), 'state.sqlite3') : ':memory:'
+    end
+
+    def source_filesystem
+      @source_filesystem ||= @options.fetch(:source_filesystem) { LocalFilesystem.new(File.expand_path(source_directory, base_directory)) }
+    end
+
+    def destination_filesystem
+      @destination_filesystem ||= @options.fetch(:destination_filesystem) { LocalFilesystem.new(File.expand_path(destination_directory, base_directory)) }
+    end
+
+    def loader
+      @loader ||= LoaderChain.new(
+        loaders: [MarkdownLoader],
+        decorators: metadata_decorators,
+      )
+    end
+
+    def logger
+      @logger ||= Logger.new($stdout)
+    end
+  end
+end