textus 0.54.2 → 0.55.2

data/lib/textus/store/freshness/ttl_evaluator.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Freshness
+      class TtlEvaluator
+        def initialize(manifest:, file_stat:, clock:)
+          @manifest  = manifest
+          @file_stat = file_stat
+          @clock     = clock
+        end
+
+        def verdict(mentry)
+          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
+          return fresh if ttl.nil?
+
+          stale = age_stale?(file_basis(mentry), ttl)
+          Verdict.build(stale: stale, reason: stale ? "ttl exceeded" : nil, fetching: false)
+        end
+
+        def stale_keys(prefix: nil, lane: nil)
+          @manifest.data.entries.select { |m| due?(m, prefix: prefix, lane: lane) }.map(&:key)
+        end
+
+        private
+
+        def fresh = Verdict.build(stale: false, reason: nil, fetching: false)
+
+        def file_basis(mentry)
+          path = @manifest.resolver.resolve(mentry.key).path
+          return nil unless @file_stat.exists?(path)
+
+          @file_stat.mtime(path)
+        end
+
+        def due?(mentry, prefix:, lane:)
+          return false if lane && mentry.lane != lane
+          return false if prefix && !mentry.key.start_with?(prefix)
+
+          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
+          return false if ttl.nil?
+
+          path = @manifest.resolver.resolve(mentry.key).path
+          return true unless @file_stat.exists?(path)
+
+          age_stale?(file_basis(mentry), ttl)
+        end
+
+        def age_stale?(basis, ttl)
+          return true if basis.nil?
+
+          (@clock.now - basis).to_i > ttl
+        end
+      end
+    end
+  end
+end

data/lib/textus/{core → store}/freshness/verdict.rb

@@ -1,16 +1,6 @@
-# frozen_string_literal: true
-
 module Textus
-  module Core
+  class Store
     module Freshness
-      # Value object describing the freshness annotation attached to an Envelope
-      # after a currency evaluation (ADR 0099 — was Core::Freshness).
-      #
-      # Note on wire format: `#to_h_for_wire` is intentionally narrower than the
-      # full field set. It emits the legacy keys ("stale", "stale_reason",
-      # "fetching", and "fetch_error" when present) so the CLI JSON wire stays
-      # byte-identical with textus/4. The gem-side fields `checked_at` and
-      # `ttl_remaining_ms` are NOT emitted on the wire.
       Verdict = Data.define(
         :stale, :fetching, :reason, :fetch_error, :checked_at, :ttl_remaining_ms
       ) do

data/lib/textus/store/freshness.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Freshness
+    end
+  end
+end

data/lib/textus/store/index/builder.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Textus
+  class Store
+    module Index
+      class Builder
+        def initialize(store:)
+          @store = store
+        end
+
+        def rebuild!(resolver:)
+          rows = resolver.enumerate.filter_map { |row| build_row(row) }
+          now_iso = Time.now.utc.iso8601
+
+          @store.transaction do
+            @store.execute("DELETE FROM entries")
+            rows.each do |data|
+              @store.execute(
+                "INSERT INTO entries (key, lane, format, etag, content, extra, indexed_at, schema_ref)
+               VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                [data[:key], data[:lane], data[:format], data[:etag],
+                 data[:content], data[:extra], now_iso, data[:schema_ref]],
+              )
+            end
+            @store.execute("INSERT INTO entries_fts(entries_fts) VALUES('rebuild')")
+          end
+          { indexed: rows.size }
+        end
+
+        private
+
+        def build_row(row)
+          key = row.fetch(:key)
+          path = row.fetch(:path)
+          entry = row.fetch(:manifest_entry)
+          return nil unless path && File.file?(path)
+
+          raw = File.read(path)
+          parsed = Textus::Format.for(entry.format).parse(raw, path: path)
+          {
+            key: key,
+            lane: entry.lane,
+            format: entry.format.to_s,
+            etag: Textus::Value::Etag.for_bytes(raw),
+            content: content_text(parsed),
+            extra: extra_json(parsed),
+            schema_ref: entry.schema,
+          }
+        end
+
+        def content_text(parsed)
+          content = parsed["content"]
+          body = parsed["body"]
+          parts = []
+          parts << body if body
+          parts << JSON.dump(content) if content
+          parts.join("\n")
+        end
+
+        def extra_json(parsed)
+          content = parsed["content"]
+          extra = {}
+          if content.is_a?(Hash)
+            extra["content_hash"] = content["content_hash"] if content["content_hash"]
+            url = content.dig("source", "url")
+            extra["url"] = url if url
+          end
+          JSON.dump(extra)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/index/lookup.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Textus
+  class Store
+    module Index
+      class Lookup
+        def initialize(store:)
+          @store = store
+        end
+
+        def search(query, lane: nil)
+          return [] if query.to_s.strip.empty?
+
+          clauses = ["entries_fts MATCH ?"]
+          params = [query]
+          if lane
+            clauses << "entries.lane = ?"
+            params << lane
+          end
+          conditions = "WHERE #{clauses.join(" AND ")}"
+          @store.execute(
+            "SELECT entries.key, entries.lane, entries.format, entries.etag, bm25(entries_fts) AS rank
+             FROM entries_fts JOIN entries ON entries_fts.rowid = entries.rowid
+           #{conditions}
+           ORDER BY rank",
+            params,
+          )
+        rescue SQLite3::SQLException
+          []
+        end
+
+        def find_by_hash(content_hash)
+          return nil if content_hash.to_s.empty?
+
+          find_extra("content_hash", content_hash)
+        end
+
+        def find_by_url(url)
+          return nil if url.to_s.empty?
+
+          find_extra("url", url)
+        end
+
+        private
+
+        def find_extra(name, value)
+          @store.execute("SELECT key, extra FROM entries ORDER BY indexed_at DESC").each do |row|
+            extra = JSON.parse(row["extra"] || "{}")
+            return row["key"] if extra[name] == value
+          end
+          nil
+        rescue SQLite3::SQLException
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/base.rb

@@ -0,0 +1,13 @@
+module Textus
+  class Store
+    module Jobs
+      class Base
+        def call(**)
+          raise NotImplementedError.new("#{self.class}#call")
+        end
+
+        def args = {}
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/index.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Index < Base
+        TYPE = "index"
+
+        def self.call(container:, call:) # rubocop:disable Lint/UnusedMethodArgument
+          Textus::Store::Index::Builder.new(store: container.job_store).rebuild!(resolver: container.manifest.resolver)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/materialize.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Materialize < Base
+        TYPE = "materialize"
+
+        def self.call(container:, call:, key:)
+          Textus::Produce::Engine.converge(container: container, call: call, keys: [key])
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/plan.rb

@@ -0,0 +1,11 @@
+module Textus
+  class Store
+    module Jobs
+      Plan = Data.define(:steps, :warnings) do
+        def to_h
+          { "steps" => steps, "warnings" => warnings }
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/planner.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Planner
+        ACTIONS_BY_TRIGGER = {
+          "convergence" => %w[materialize sweep index],
+          "entry.written" => %w[materialize],
+          "entry.deleted" => %w[materialize],
+          "entry.moved" => %w[materialize],
+          "proposal.accepted" => %w[materialize],
+          "proposal.rejected" => %w[materialize],
+        }.freeze
+
+        ENTRY_LEVEL_TRIGGERS = %w[
+          entry.written entry.deleted entry.moved
+          proposal.accepted proposal.rejected
+        ].freeze
+
+        SCOPE_RESOLVERS = {
+          "materialize" => :producible_keys,
+          "sweep" => :lane_keys,
+        }.freeze
+
+        GLOBAL_ACTIONS = {
+          "index" => {},
+          "sweep" => { "scope" => {} },
+        }.freeze
+
+        def self.seed(container:, queue:, role:)
+          jobs = new(container: container).plan(
+            trigger: { "type" => "convergence" },
+            role: role,
+          )
+          jobs.each { |j| queue.enqueue(j) }
+        end
+
+        def initialize(container:)
+          @container = container
+          @manifest  = container.manifest
+        end
+
+        def plan(trigger:, role:)
+          type   = trigger["type"] || trigger[:type]
+          target = trigger["target"] || trigger[:target]
+          return [] if type.nil?
+
+          blocks_with_react = @manifest.rules.blocks.select(&:react)
+          if blocks_with_react.any?
+            plan_from_rules(blocks_with_react, type, role, target: target)
+          else
+            plan_from_defaults(type, role, target: target)
+          end
+        end
+
+        private
+
+        def plan_from_rules(blocks, type, role, target: nil) # rubocop:disable Lint/UnusedMethodArgument
+          jobs = []
+          blocks
+            .select { |b| matches_trigger?(b.react, type) }
+            .each do |block|
+              do_action = block.react.raw["do"]
+              Array(do_action).each do |action|
+                if (global_args = GLOBAL_ACTIONS[action])
+                  jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: global_args, role: role)
+                else
+                  resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
+                  keys = send(resolver, nil)
+                  keys.each { |key| jobs << job(action, key, role) }
+                end
+              end
+            end
+          jobs
+        end
+
+        def plan_from_defaults(type, role, target: nil)
+          actions = ACTIONS_BY_TRIGGER.fetch(type, [])
+          jobs = []
+          if actions.include?("materialize")
+            keys = target && ENTRY_LEVEL_TRIGGERS.include?(type) ? dependent_keys(target) : producible_keys(nil)
+            keys.each { |k| jobs << job("materialize", k, role) }
+          end
+          GLOBAL_ACTIONS.each do |action, args|
+            jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: args, role: role) if actions.include?(action)
+          end
+          jobs
+        end
+
+        def matches_trigger?(react, type)
+          on = react.raw["on"]
+          Array(on).include?(type)
+        end
+
+        def job(type, key, role)
+          Textus::Store::Jobs::Queue::Job.new(type: type, args: { "key" => key }, role: role)
+        end
+
+        def producible_keys(_target)
+          @manifest.data.entries
+                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                   .map(&:key)
+        end
+
+        def lane_keys(_target)
+          @manifest.data.entries.map(&:key)
+        end
+
+        def dependent_keys(target_key)
+          @manifest.data.entries
+                   .select(&:external?)
+                   .select do |e|
+                     Array(e.source&.sources).compact.any? do |s|
+                       target_key == s || target_key.start_with?("#{s}.")
+                     end
+                   end
+                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                   .map(&:key)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/queue.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "securerandom"
+require "time"
+
+module Textus
+  class Store
+    module Jobs
+      class Queue
+        VALID_STATES = %w[ready leased done failed].freeze
+
+        Leased = Data.define(:job)
+
+        class Job
+          DIGEST_BYTES = 16
+
+          attr_reader :type, :args, :role, :attempts, :max_attempts, :errors
+
+          def initialize(type:, args:, role:, attempts: 0, max_attempts: 3, errors: [])
+            @type = type.to_s
+            @args = stringify(args)
+            @role = role.to_s
+            @attempts = attempts.to_i
+            @max_attempts = max_attempts.to_i
+            @errors = Array(errors)
+          end
+
+          def id
+            "#{type}:#{Digest::SHA256.hexdigest(JSON.dump(args.sort.to_h))[0, DIGEST_BYTES]}"
+          end
+
+          private
+
+          def stringify(hash)
+            hash.to_h.transform_keys(&:to_s)
+          end
+        end
+
+        def initialize(store:)
+          @store = store
+        end
+
+        def enqueue(job)
+          now = iso_now
+          inserted = @store.execute(
+            "INSERT OR IGNORE INTO jobs (id, type, args, state, role, attempts, max_attempts, errors, lease, created_at, updated_at)
+           VALUES (?, ?, ?, 'ready', ?, ?, ?, ?, NULL, ?, ?)",
+            [job.id, job.type, JSON.dump(job.args), job.role, job.attempts, job.max_attempts, JSON.dump(job.errors), now, now],
+          )
+          return inserted if @store.query_value("SELECT changes()")&.to_i&.positive?
+
+          @store.execute(
+            "UPDATE jobs
+               SET state = 'ready', role = ?, args = ?, attempts = 0, errors = ?, lease = NULL, max_attempts = ?, updated_at = ?
+             WHERE id = ? AND state IN ('done', 'failed')",
+            [job.role, JSON.dump(job.args), JSON.dump([]), job.max_attempts, now, job.id],
+          )
+        end
+
+        def ready_ids
+          list(:ready)
+        end
+
+        def lease(worker_id:, lease_ttl:)
+          now = Time.now.utc
+          expires_at = now + lease_ttl
+          token = SecureRandom.hex(8)
+          marked_lease = JSON.dump({ "worker_id" => worker_id, "expires_at" => expires_at.iso8601, "token" => token })
+
+          @store.execute(
+            "UPDATE jobs
+              SET state = 'leased', lease = ?, updated_at = ?
+            WHERE id = (
+              SELECT id FROM jobs WHERE state = 'ready' ORDER BY created_at, id LIMIT 1
+            )",
+            [marked_lease, now.iso8601],
+          )
+          row = @store.execute("SELECT * FROM jobs WHERE state = 'leased' AND lease = ? LIMIT 1", [marked_lease]).first
+          return nil unless row
+
+          Leased.new(job_from_row(row))
+        end
+
+        def ack(leased)
+          @store.execute(
+            "UPDATE jobs SET state = 'done', lease = NULL, updated_at = ? WHERE id = ? AND state = 'leased'",
+            [iso_now, leased.job.id],
+          )
+        end
+
+        def fail(leased, error:)
+          job = leased.job
+          attempts = job.attempts + 1
+          errors = job.errors + [{ "attempt" => attempts, "error" => error, "at" => iso_now }]
+          dead = attempts >= job.max_attempts
+          state = dead ? "failed" : "ready"
+          @store.execute(
+            "UPDATE jobs SET state = ?, attempts = ?, errors = ?, lease = NULL, updated_at = ? WHERE id = ?",
+            [state, attempts, JSON.dump(errors), iso_now, job.id],
+          )
+          dead ? :dead_lettered : :requeued
+        end
+
+        def reclaim(now:)
+          rows = @store.execute("SELECT id, lease FROM jobs WHERE state = 'leased'")
+          expired = rows.select do |row|
+            lease = JSON.parse(row["lease"] || "{}")
+            expires_at = lease["expires_at"]
+            expires_at.nil? || Time.parse(expires_at) <= now
+          end
+          expired.each do |row|
+            @store.execute(
+              "UPDATE jobs SET state = 'ready', lease = NULL, updated_at = ? WHERE id = ?",
+              [now.utc.iso8601, row["id"]],
+            )
+          end
+          expired.size
+        end
+
+        def list(state)
+          state = state.to_s
+          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
+
+          @store.execute("SELECT id FROM jobs WHERE state = ? ORDER BY created_at, id", [state]).map { |row| row["id"] }
+        end
+
+        def retry_failed(job_id)
+          @store.execute(
+            "UPDATE jobs SET state = 'ready', attempts = 0, errors = ?, lease = NULL, updated_at = ? WHERE id = ? AND state = 'failed'",
+            [JSON.dump([]), iso_now, job_id],
+          )
+        end
+
+        def purge(state)
+          state = state.to_s
+          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
+
+          @store.execute("DELETE FROM jobs WHERE state = ?", [state])
+        end
+
+        private
+
+        def job_from_row(row)
+          Job.new(
+            type: row["type"],
+            args: JSON.parse(row["args"] || "{}"),
+            role: row["role"],
+            attempts: row["attempts"],
+            max_attempts: row["max_attempts"],
+            errors: JSON.parse(row["errors"] || "[]"),
+          )
+        end
+
+        def iso_now
+          Time.now.utc.iso8601
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/registry.rb

@@ -0,0 +1,19 @@
+module Textus
+  class Store
+    module Jobs
+      module Registry
+        class UnknownJob < KeyError; end
+
+        JOBS = {
+          "index" => Store::Jobs::Index,
+          "materialize" => Store::Jobs::Materialize,
+          "sweep" => Store::Jobs::Sweep,
+        }.freeze
+
+        def self.fetch(type)
+          JOBS.fetch(type.to_s) { raise UnknownJob.new("Unknown job type: #{type}") }
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/retention/base.rb

@@ -0,0 +1,52 @@
+require "fileutils"
+
+module Textus
+  class Store
+    module Jobs
+      module Retention
+        class Base
+          def initialize(container:, call:)
+            @container = container
+            @call = call
+          end
+
+          def call(rows)
+            out = { dropped: [], archived: [], failed: [] }
+            rows.each do |row|
+              key = row["key"]
+              begin
+                case row["action"]
+                when "drop"
+                  delete(key)
+                  out[:dropped] << key
+                when "archive"
+                  archive_leaf(row)
+                  delete(key)
+                  out[:archived] << key
+                end
+              rescue Textus::Error => e
+                out[:failed] << { "key" => key, "error" => e.message }
+              end
+            end
+            out
+          end
+
+          private
+
+          def archive_leaf(row)
+            src  = row["path"]
+            dest = @container.layout.archive_path(src)
+            FileUtils.mkdir_p(File.dirname(dest))
+            FileUtils.cp(src, dest)
+          end
+
+          def delete(key)
+            mentry = @container.manifest.resolver.resolve(key).entry
+            writer = Textus::Store::Entry::Writer.from(container: @container, call: @call)
+            writer.delete(key, mentry: mentry)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/retention/sweep.rb

@@ -0,0 +1,55 @@
+require "time"
+
+module Textus
+  class Store
+    module Jobs
+      module Retention
+        class Sweep
+          def self.expired?(ttl_seconds:, mtime:, now:)
+            return false if ttl_seconds.nil? || mtime.nil?
+
+            (now - mtime).to_i > ttl_seconds
+          end
+
+          def initialize(manifest:, file_stat:, clock:)
+            @manifest = manifest
+            @file_stat = file_stat
+            @clock = clock
+          end
+
+          def call(prefix: nil, lane: nil)
+            @manifest.data.entries
+                     .select { |m| matches?(m, prefix: prefix, lane: lane) }
+                     .flat_map { |m| rows_for(m) }
+          end
+
+          private
+
+          def matches?(mentry, prefix:, lane:)
+            return false if lane && mentry.lane != lane
+            return false if prefix && !Textus::Key::Matching.matches_prefix?(
+              mentry.key, prefix, nested: mentry.is_a?(Textus::Manifest::Entry::Nested)
+            )
+
+            true
+          end
+
+          def rows_for(mentry)
+            policy = @manifest.rules.for(mentry.key).retention
+            return [] if policy.nil?
+
+            @manifest.resolver.enumerate(prefix: mentry.key).filter_map do |row|
+              path = row[:path]
+              next unless @file_stat.exists?(path)
+              next unless self.class.expired?(
+                ttl_seconds: policy.ttl_seconds, mtime: @file_stat.mtime(path), now: @clock.now,
+              )
+
+              { "key" => row[:key], "path" => path, "action" => policy.action.to_s }
+            end
+          end
+        end
+      end
+    end
+  end
+end