textus 0.55.2 → 0.56.0

data/lib/textus/store/freshness/drift_detector.rb

@@ -1,93 +0,0 @@
-# frozen_string_literal: true
-
-require "time"
-
-module Textus
-  class Store
-    module Freshness
-      class DriftDetector
-        def initialize(manifest:, file_stat:, clock:)
-          @manifest  = manifest
-          @file_stat = file_stat
-          @clock     = clock
-        end
-
-        def drift_rows(mentry)
-          return [] unless mentry.external?
-
-          path   = Textus::Key::Path.resolve(@manifest.data, mentry)
-          reason = drift_reason(mentry, path)
-          reason ? [drift_row(mentry, path, reason)] : []
-        end
-
-        private
-
-        def drift_reason(mentry, path)
-          return "derived entry has never been generated" unless @file_stat.exists?(path)
-
-          generated_at = generated_at_of(mentry, path)
-          return "missing generated.at frontmatter" unless generated_at
-
-          gen_time = parse_time(generated_at)
-          return "unparseable generated.at: #{generated_at.inspect}" unless gen_time
-
-          offender = newest_source_after(mentry.source, gen_time)
-          "source '#{offender}' modified after generated.at" if offender
-        end
-
-        def generated_at_of(mentry, path)
-          Textus::Format.for(mentry.format).parse(@file_stat.read(path), path: path)["_meta"]
-                        .dig("generated", "at")
-        end
-
-        def parse_time(str)
-          Time.parse(str.to_s)
-        rescue StandardError
-          nil
-        end
-
-        def newest_source_after(external_src, gen_time)
-          Array(external_src.sources).each do |src|
-            offender = check_source(src, gen_time)
-            return offender if offender
-          end
-          nil
-        end
-
-        def check_source(src, gen_time)
-          if src.match?(/\A[a-z0-9.][a-z0-9._-]*\z/) && !src.include?("/")
-            @manifest.resolver.enumerate(prefix: src).each do |row|
-              return src if @file_stat.mtime(row[:path]) > gen_time
-            end
-            nil
-          else
-            check_filesystem_source(src, gen_time)
-          end
-        end
-
-        def check_filesystem_source(src, gen_time)
-          abs = absolutize_source(src)
-          if @file_stat.directory?(abs)
-            dir_has_newer_file?(abs, gen_time) ? src : nil
-          elsif @file_stat.exists?(abs) && @file_stat.mtime(abs) > gen_time
-            src
-          end
-        end
-
-        def absolutize_source(src)
-          File.absolute_path?(src) ? src : File.join(File.dirname(@manifest.data.root), src)
-        end
-
-        def dir_has_newer_file?(abs, gen_time)
-          @file_stat.glob(File.join(abs, "**", "*")).any? do |fpath|
-            !@file_stat.directory?(fpath) && @file_stat.exists?(fpath) && @file_stat.mtime(fpath) > gen_time
-          end
-        end
-
-        def drift_row(mentry, path, reason)
-          { "key" => mentry.key, "path" => path, "generator" => mentry.source.command, "reason" => reason }
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/freshness/evaluator.rb

@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Freshness
-      # Thin facade delegating to the focused TtlEvaluator and DriftDetector.
-      # Prefer using TtlEvaluator or DriftDetector directly.
-      class Evaluator
-        def initialize(manifest:, file_stat:, clock:)
-          @ttl   = TtlEvaluator.new(manifest: manifest, file_stat: file_stat, clock: clock)
-          @drift = DriftDetector.new(manifest: manifest, file_stat: file_stat, clock: clock)
-        end
-
-        def verdict(mentry) = @ttl.verdict(mentry)
-        def stale_keys(**) = @ttl.stale_keys(**)
-        def drift_rows(mentry) = @drift.drift_rows(mentry)
-      end
-    end
-  end
-end

data/lib/textus/store/freshness/verdict.rb

@@ -1,24 +0,0 @@
-module Textus
-  class Store
-    module Freshness
-      Verdict = Data.define(
-        :stale, :fetching, :reason, :fetch_error, :checked_at, :ttl_remaining_ms
-      ) do
-        def self.build(stale:, fetching: false, reason: nil, fetch_error: nil,
-                       checked_at: nil, ttl_remaining_ms: nil)
-          new(
-            stale: stale, fetching: fetching, reason: reason,
-            fetch_error: fetch_error, checked_at: checked_at,
-            ttl_remaining_ms: ttl_remaining_ms
-          )
-        end
-
-        def to_h_for_wire
-          h = { "stale" => stale, "stale_reason" => reason, "fetching" => fetching }
-          h["fetch_error"] = fetch_error unless fetch_error.nil?
-          h
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/freshness.rb

@@ -1,8 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Freshness
-    end
-  end
-end

data/lib/textus/store/index/builder.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "time"
-
-module Textus
-  class Store
-    module Index
-      class Builder
-        def initialize(store:)
-          @store = store
-        end
-
-        def rebuild!(resolver:)
-          rows = resolver.enumerate.filter_map { |row| build_row(row) }
-          now_iso = Time.now.utc.iso8601
-
-          @store.transaction do
-            @store.execute("DELETE FROM entries")
-            rows.each do |data|
-              @store.execute(
-                "INSERT INTO entries (key, lane, format, etag, content, extra, indexed_at, schema_ref)
-               VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
-                [data[:key], data[:lane], data[:format], data[:etag],
-                 data[:content], data[:extra], now_iso, data[:schema_ref]],
-              )
-            end
-            @store.execute("INSERT INTO entries_fts(entries_fts) VALUES('rebuild')")
-          end
-          { indexed: rows.size }
-        end
-
-        private
-
-        def build_row(row)
-          key = row.fetch(:key)
-          path = row.fetch(:path)
-          entry = row.fetch(:manifest_entry)
-          return nil unless path && File.file?(path)
-
-          raw = File.read(path)
-          parsed = Textus::Format.for(entry.format).parse(raw, path: path)
-          {
-            key: key,
-            lane: entry.lane,
-            format: entry.format.to_s,
-            etag: Textus::Value::Etag.for_bytes(raw),
-            content: content_text(parsed),
-            extra: extra_json(parsed),
-            schema_ref: entry.schema,
-          }
-        end
-
-        def content_text(parsed)
-          content = parsed["content"]
-          body = parsed["body"]
-          parts = []
-          parts << body if body
-          parts << JSON.dump(content) if content
-          parts.join("\n")
-        end
-
-        def extra_json(parsed)
-          content = parsed["content"]
-          extra = {}
-          if content.is_a?(Hash)
-            extra["content_hash"] = content["content_hash"] if content["content_hash"]
-            url = content.dig("source", "url")
-            extra["url"] = url if url
-          end
-          JSON.dump(extra)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/index/lookup.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Textus
-  class Store
-    module Index
-      class Lookup
-        def initialize(store:)
-          @store = store
-        end
-
-        def search(query, lane: nil)
-          return [] if query.to_s.strip.empty?
-
-          clauses = ["entries_fts MATCH ?"]
-          params = [query]
-          if lane
-            clauses << "entries.lane = ?"
-            params << lane
-          end
-          conditions = "WHERE #{clauses.join(" AND ")}"
-          @store.execute(
-            "SELECT entries.key, entries.lane, entries.format, entries.etag, bm25(entries_fts) AS rank
-             FROM entries_fts JOIN entries ON entries_fts.rowid = entries.rowid
-           #{conditions}
-           ORDER BY rank",
-            params,
-          )
-        rescue SQLite3::SQLException
-          []
-        end
-
-        def find_by_hash(content_hash)
-          return nil if content_hash.to_s.empty?
-
-          find_extra("content_hash", content_hash)
-        end
-
-        def find_by_url(url)
-          return nil if url.to_s.empty?
-
-          find_extra("url", url)
-        end
-
-        private
-
-        def find_extra(name, value)
-          @store.execute("SELECT key, extra FROM entries ORDER BY indexed_at DESC").each do |row|
-            extra = JSON.parse(row["extra"] || "{}")
-            return row["key"] if extra[name] == value
-          end
-          nil
-        rescue SQLite3::SQLException
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/base.rb

@@ -1,13 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      class Base
-        def call(**)
-          raise NotImplementedError.new("#{self.class}#call")
-        end
-
-        def args = {}
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/index.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Index < Base
-        TYPE = "index"
-
-        def self.call(container:, call:) # rubocop:disable Lint/UnusedMethodArgument
-          Textus::Store::Index::Builder.new(store: container.job_store).rebuild!(resolver: container.manifest.resolver)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/materialize.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Materialize < Base
-        TYPE = "materialize"
-
-        def self.call(container:, call:, key:)
-          Textus::Produce::Engine.converge(container: container, call: call, keys: [key])
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/plan.rb

@@ -1,11 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      Plan = Data.define(:steps, :warnings) do
-        def to_h
-          { "steps" => steps, "warnings" => warnings }
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/planner.rb

@@ -1,124 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Planner
-        ACTIONS_BY_TRIGGER = {
-          "convergence" => %w[materialize sweep index],
-          "entry.written" => %w[materialize],
-          "entry.deleted" => %w[materialize],
-          "entry.moved" => %w[materialize],
-          "proposal.accepted" => %w[materialize],
-          "proposal.rejected" => %w[materialize],
-        }.freeze
-
-        ENTRY_LEVEL_TRIGGERS = %w[
-          entry.written entry.deleted entry.moved
-          proposal.accepted proposal.rejected
-        ].freeze
-
-        SCOPE_RESOLVERS = {
-          "materialize" => :producible_keys,
-          "sweep" => :lane_keys,
-        }.freeze
-
-        GLOBAL_ACTIONS = {
-          "index" => {},
-          "sweep" => { "scope" => {} },
-        }.freeze
-
-        def self.seed(container:, queue:, role:)
-          jobs = new(container: container).plan(
-            trigger: { "type" => "convergence" },
-            role: role,
-          )
-          jobs.each { |j| queue.enqueue(j) }
-        end
-
-        def initialize(container:)
-          @container = container
-          @manifest  = container.manifest
-        end
-
-        def plan(trigger:, role:)
-          type   = trigger["type"] || trigger[:type]
-          target = trigger["target"] || trigger[:target]
-          return [] if type.nil?
-
-          blocks_with_react = @manifest.rules.blocks.select(&:react)
-          if blocks_with_react.any?
-            plan_from_rules(blocks_with_react, type, role, target: target)
-          else
-            plan_from_defaults(type, role, target: target)
-          end
-        end
-
-        private
-
-        def plan_from_rules(blocks, type, role, target: nil) # rubocop:disable Lint/UnusedMethodArgument
-          jobs = []
-          blocks
-            .select { |b| matches_trigger?(b.react, type) }
-            .each do |block|
-              do_action = block.react.raw["do"]
-              Array(do_action).each do |action|
-                if (global_args = GLOBAL_ACTIONS[action])
-                  jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: global_args, role: role)
-                else
-                  resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
-                  keys = send(resolver, nil)
-                  keys.each { |key| jobs << job(action, key, role) }
-                end
-              end
-            end
-          jobs
-        end
-
-        def plan_from_defaults(type, role, target: nil)
-          actions = ACTIONS_BY_TRIGGER.fetch(type, [])
-          jobs = []
-          if actions.include?("materialize")
-            keys = target && ENTRY_LEVEL_TRIGGERS.include?(type) ? dependent_keys(target) : producible_keys(nil)
-            keys.each { |k| jobs << job("materialize", k, role) }
-          end
-          GLOBAL_ACTIONS.each do |action, args|
-            jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: args, role: role) if actions.include?(action)
-          end
-          jobs
-        end
-
-        def matches_trigger?(react, type)
-          on = react.raw["on"]
-          Array(on).include?(type)
-        end
-
-        def job(type, key, role)
-          Textus::Store::Jobs::Queue::Job.new(type: type, args: { "key" => key }, role: role)
-        end
-
-        def producible_keys(_target)
-          @manifest.data.entries
-                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
-                   .map(&:key)
-        end
-
-        def lane_keys(_target)
-          @manifest.data.entries.map(&:key)
-        end
-
-        def dependent_keys(target_key)
-          @manifest.data.entries
-                   .select(&:external?)
-                   .select do |e|
-                     Array(e.source&.sources).compact.any? do |s|
-                       target_key == s || target_key.start_with?("#{s}.")
-                     end
-                   end
-                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
-                   .map(&:key)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/queue.rb

@@ -1,162 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-require "json"
-require "securerandom"
-require "time"
-
-module Textus
-  class Store
-    module Jobs
-      class Queue
-        VALID_STATES = %w[ready leased done failed].freeze
-
-        Leased = Data.define(:job)
-
-        class Job
-          DIGEST_BYTES = 16
-
-          attr_reader :type, :args, :role, :attempts, :max_attempts, :errors
-
-          def initialize(type:, args:, role:, attempts: 0, max_attempts: 3, errors: [])
-            @type = type.to_s
-            @args = stringify(args)
-            @role = role.to_s
-            @attempts = attempts.to_i
-            @max_attempts = max_attempts.to_i
-            @errors = Array(errors)
-          end
-
-          def id
-            "#{type}:#{Digest::SHA256.hexdigest(JSON.dump(args.sort.to_h))[0, DIGEST_BYTES]}"
-          end
-
-          private
-
-          def stringify(hash)
-            hash.to_h.transform_keys(&:to_s)
-          end
-        end
-
-        def initialize(store:)
-          @store = store
-        end
-
-        def enqueue(job)
-          now = iso_now
-          inserted = @store.execute(
-            "INSERT OR IGNORE INTO jobs (id, type, args, state, role, attempts, max_attempts, errors, lease, created_at, updated_at)
-           VALUES (?, ?, ?, 'ready', ?, ?, ?, ?, NULL, ?, ?)",
-            [job.id, job.type, JSON.dump(job.args), job.role, job.attempts, job.max_attempts, JSON.dump(job.errors), now, now],
-          )
-          return inserted if @store.query_value("SELECT changes()")&.to_i&.positive?
-
-          @store.execute(
-            "UPDATE jobs
-               SET state = 'ready', role = ?, args = ?, attempts = 0, errors = ?, lease = NULL, max_attempts = ?, updated_at = ?
-             WHERE id = ? AND state IN ('done', 'failed')",
-            [job.role, JSON.dump(job.args), JSON.dump([]), job.max_attempts, now, job.id],
-          )
-        end
-
-        def ready_ids
-          list(:ready)
-        end
-
-        def lease(worker_id:, lease_ttl:)
-          now = Time.now.utc
-          expires_at = now + lease_ttl
-          token = SecureRandom.hex(8)
-          marked_lease = JSON.dump({ "worker_id" => worker_id, "expires_at" => expires_at.iso8601, "token" => token })
-
-          @store.execute(
-            "UPDATE jobs
-              SET state = 'leased', lease = ?, updated_at = ?
-            WHERE id = (
-              SELECT id FROM jobs WHERE state = 'ready' ORDER BY created_at, id LIMIT 1
-            )",
-            [marked_lease, now.iso8601],
-          )
-          row = @store.execute("SELECT * FROM jobs WHERE state = 'leased' AND lease = ? LIMIT 1", [marked_lease]).first
-          return nil unless row
-
-          Leased.new(job_from_row(row))
-        end
-
-        def ack(leased)
-          @store.execute(
-            "UPDATE jobs SET state = 'done', lease = NULL, updated_at = ? WHERE id = ? AND state = 'leased'",
-            [iso_now, leased.job.id],
-          )
-        end
-
-        def fail(leased, error:)
-          job = leased.job
-          attempts = job.attempts + 1
-          errors = job.errors + [{ "attempt" => attempts, "error" => error, "at" => iso_now }]
-          dead = attempts >= job.max_attempts
-          state = dead ? "failed" : "ready"
-          @store.execute(
-            "UPDATE jobs SET state = ?, attempts = ?, errors = ?, lease = NULL, updated_at = ? WHERE id = ?",
-            [state, attempts, JSON.dump(errors), iso_now, job.id],
-          )
-          dead ? :dead_lettered : :requeued
-        end
-
-        def reclaim(now:)
-          rows = @store.execute("SELECT id, lease FROM jobs WHERE state = 'leased'")
-          expired = rows.select do |row|
-            lease = JSON.parse(row["lease"] || "{}")
-            expires_at = lease["expires_at"]
-            expires_at.nil? || Time.parse(expires_at) <= now
-          end
-          expired.each do |row|
-            @store.execute(
-              "UPDATE jobs SET state = 'ready', lease = NULL, updated_at = ? WHERE id = ?",
-              [now.utc.iso8601, row["id"]],
-            )
-          end
-          expired.size
-        end
-
-        def list(state)
-          state = state.to_s
-          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
-
-          @store.execute("SELECT id FROM jobs WHERE state = ? ORDER BY created_at, id", [state]).map { |row| row["id"] }
-        end
-
-        def retry_failed(job_id)
-          @store.execute(
-            "UPDATE jobs SET state = 'ready', attempts = 0, errors = ?, lease = NULL, updated_at = ? WHERE id = ? AND state = 'failed'",
-            [JSON.dump([]), iso_now, job_id],
-          )
-        end
-
-        def purge(state)
-          state = state.to_s
-          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
-
-          @store.execute("DELETE FROM jobs WHERE state = ?", [state])
-        end
-
-        private
-
-        def job_from_row(row)
-          Job.new(
-            type: row["type"],
-            args: JSON.parse(row["args"] || "{}"),
-            role: row["role"],
-            attempts: row["attempts"],
-            max_attempts: row["max_attempts"],
-            errors: JSON.parse(row["errors"] || "[]"),
-          )
-        end
-
-        def iso_now
-          Time.now.utc.iso8601
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/registry.rb

@@ -1,19 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      module Registry
-        class UnknownJob < KeyError; end
-
-        JOBS = {
-          "index" => Store::Jobs::Index,
-          "materialize" => Store::Jobs::Materialize,
-          "sweep" => Store::Jobs::Sweep,
-        }.freeze
-
-        def self.fetch(type)
-          JOBS.fetch(type.to_s) { raise UnknownJob.new("Unknown job type: #{type}") }
-        end
-      end
-    end
-  end
-end