textus 0.55.1 → 0.56.0

data/lib/textus/store/envelope/writer.rb

@@ -1,195 +0,0 @@
-require "fileutils"
-
-module Textus
-  class Store
-    module Envelope
-      # Owns the write pipeline (validate, serialize, etag-check, write, audit).
-      # Talks to ports (FileStore, Schemas, AuditLog, Manifest) and an
-      # Reader for the existing-uid lookup.
-      #
-      # Invariant: every public method's final action is @audit_log.append(...).
-      #
-      # No permission check, no event firing — those belong to the caller
-      # (Write::Put / ::Delete / ::Mv).
-      class Writer
-        Payload = Data.define(:meta, :body, :content)
-
-        def self.from(container:, call:)
-          new(
-            file_store: container.file_store, manifest: container.manifest,
-            schemas: container.schemas, audit_log: container.audit_log,
-            call: call, reader: Reader.from(container: container),
-            geometry: container.geometry
-          )
-        end
-
-        def initialize(file_store:, manifest:, schemas:, audit_log:, call:, reader:, geometry:) # rubocop:disable Metrics/ParameterLists
-          @file_store = file_store
-          @manifest   = manifest
-          @schemas    = schemas
-          @audit_log  = audit_log
-          @call       = call
-          @reader     = reader
-          @geometry   = geometry
-        end
-
-        def put(key, mentry:, payload:, if_etag: nil)
-          path = resolve_path(key)
-          meta = payload.meta || {}
-          content = payload.content
-          existing_env = @reader.read(key)
-          existing_meta = existing_env ? existing_env.meta : {}
-          meta, content = Textus::Meta.inject_all(meta, content, existing_meta, format: mentry.format)
-          bytes, eff_meta, eff_body, eff_content = serialize_entry(mentry, path, meta, payload, content)
-          enforce_name_match!(path, eff_meta, mentry.format)
-          validate_schema(mentry, eff_meta, eff_content)
-          Textus::Format::Yaml.validate_raw_entry!(
-            { "_meta" => eff_meta, "content" => eff_content },
-            mentry.lane,
-          )
-          etag_before = check_etag!(path, key, if_etag)
-          write_bytes(path, bytes)
-          envelope = build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes)
-          audit_put(key, etag_before, envelope.etag)
-          envelope
-        end
-
-        def delete(key, mentry: nil, if_etag: nil) # rubocop:disable Lint/UnusedMethodArgument
-          # `mentry:` is accepted for symmetry with `put` / `move` and to
-          # leave room for future format-specific delete hooks; no field
-          # on it is needed today.
-          path = @manifest.resolver.resolve(key).path
-          raise UnknownKey.new(key, suggestions: @manifest.resolver.suggestions_for(key)) unless @file_store.exists?(path)
-
-          etag_before = @file_store.etag(path)
-          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && if_etag != etag_before
-
-          @file_store.delete(path)
-          prune_empty_parents(path)
-          @audit_log.append(
-            role: @call.role, verb: "key_delete", key: key,
-            etag_before: etag_before, etag_after: nil,
-            extras: @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
-          )
-        end
-
-        def move(from_key:, to_key:, new_mentry:, if_etag: nil)
-          from_path = @manifest.resolver.resolve(from_key).path
-          to_path   = @manifest.resolver.resolve(to_key).path
-          raise UnknownKey.new(from_key, suggestions: @manifest.resolver.suggestions_for(from_key)) unless @file_store.exists?(from_path)
-
-          etag_before = @file_store.etag(from_path)
-          raise EtagMismatch.new(from_key, if_etag, etag_before) if if_etag && if_etag != etag_before
-
-          FileUtils.mkdir_p(File.dirname(to_path))
-          FileUtils.mv(from_path, to_path)
-          prune_empty_parents(from_path)
-          basename = to_key.split(".").last
-          Format.for(new_mentry.format).rewrite_name(to_path, basename)
-          etag_after = Value::Etag.for_file(to_path)
-
-          envelope = @reader.read(to_key)
-
-          extras = {
-            "from_key" => from_key, "to_key" => to_key,
-            "from_path" => from_path, "to_path" => to_path,
-            "uid" => envelope.uid
-          }
-          extras["correlation_id"] = @call.correlation_id if @call.correlation_id
-
-          @audit_log.append(
-            role: @call.role, verb: "key_mv", key: to_key,
-            etag_before: etag_before, etag_after: etag_after,
-            extras: extras
-          )
-
-          envelope
-        end
-
-        private
-
-        # After a file leaves a directory (delete or move-source), remove any
-        # now-empty parent dirs so bulk move/delete doesn't accrue orphan dirs
-        # (F3 of #161). Floored at the entry's *zone directory* — a zone is a
-        # declared, first-class container, so its own dir is preserved even when
-        # momentarily empty; only the sub-dirs the bulk op carved out are
-        # pruned. Stops at the first non-empty ancestor, so a dir holding a
-        # `.gitkeep` or sibling entries survives. Best-effort: a lost race or a
-        # non-empty dir is silently fine, never fatal to the write.
-        def prune_empty_parents(path)
-          floor = @geometry.lane_floor(path)
-          return unless floor
-
-          dir = File.dirname(path)
-          while dir.start_with?("#{floor}/") && Dir.empty?(dir)
-            Dir.rmdir(dir)
-            dir = File.dirname(dir)
-          end
-        rescue SystemCallError
-          nil
-        end
-
-        def enforce_name_match!(path, meta, format)
-          Textus::Format.for(format).enforce_name_match!(path, meta)
-        end
-
-        def serialize_for_put(mentry:, path:, meta:, body:, content:)
-          Textus::Format.for(mentry.format).serialize_for_put(
-            meta: meta, body: body, content: content, path: path,
-          )
-        end
-
-        def resolve_path(key)
-          @manifest.resolver.resolve(key).path
-        end
-
-        def serialize_entry(mentry, path, meta, payload, content)
-          serialize_for_put(
-            mentry: mentry, path: path,
-            meta: meta, body: payload.body, content: content
-          )
-        end
-
-        def validate_schema(mentry, eff_meta, eff_content)
-          schema = @schemas.fetch_or_nil(mentry.schema)
-          return unless schema
-
-          Format.for(mentry.format).validate_against(
-            schema,
-            { "_meta" => eff_meta, "content" => eff_content },
-          )
-        end
-
-        def check_etag!(path, key, if_etag)
-          etag_before = @file_store.exists?(path) ? @file_store.etag(path) : nil
-          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
-
-          etag_before
-        end
-
-        def write_bytes(path, bytes)
-          @file_store.write(path, bytes)
-        end
-
-        def build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes = nil) # rubocop:disable Metrics/ParameterLists
-          raw = bytes || @file_store.read(path)
-          Textus::Value::Envelope.build(
-            key: key, mentry: mentry, path: path,
-            meta: eff_meta, body: eff_body,
-            etag: Value::Etag.for_bytes(raw),
-            content: eff_content
-          )
-        end
-
-        def audit_put(key, etag_before, etag_after)
-          extras = @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
-          @audit_log.append(
-            role: @call.role, verb: "put", key: key,
-            etag_before: etag_before, etag_after: etag_after,
-            extras: extras
-          )
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/index/builder.rb

@@ -1,74 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-require "time"
-
-module Textus
-  class Store
-    module Index
-      class Builder
-        def initialize(store:)
-          @store = store
-        end
-
-        def rebuild!(resolver:)
-          rows = resolver.enumerate.filter_map { |row| build_row(row) }
-          now_iso = Time.now.utc.iso8601
-
-          @store.transaction do
-            @store.execute("DELETE FROM entries")
-            rows.each do |data|
-              @store.execute(
-                "INSERT INTO entries (key, lane, format, etag, content, extra, indexed_at)
-               VALUES (?, ?, ?, ?, ?, ?, ?)",
-                [data[:key], data[:lane], data[:format], data[:etag], data[:content], data[:extra], now_iso],
-              )
-            end
-            @store.execute("INSERT INTO entries_fts(entries_fts) VALUES('rebuild')")
-          end
-          { indexed: rows.size }
-        end
-
-        private
-
-        def build_row(row)
-          key = row.fetch(:key)
-          path = row.fetch(:path)
-          entry = row.fetch(:manifest_entry)
-          return nil unless path && File.file?(path)
-
-          raw = File.read(path)
-          parsed = Textus::Format.for(entry.format).parse(raw, path: path)
-          {
-            key: key,
-            lane: entry.lane,
-            format: entry.format.to_s,
-            etag: Textus::Value::Etag.for_bytes(raw),
-            content: content_text(parsed),
-            extra: extra_json(parsed),
-          }
-        end
-
-        def content_text(parsed)
-          content = parsed["content"]
-          body = parsed["body"]
-          parts = []
-          parts << body if body
-          parts << JSON.dump(content) if content
-          parts.join("\n")
-        end
-
-        def extra_json(parsed)
-          content = parsed["content"]
-          extra = {}
-          if content.is_a?(Hash)
-            extra["content_hash"] = content["content_hash"] if content["content_hash"]
-            url = content.dig("source", "url")
-            extra["url"] = url if url
-          end
-          JSON.dump(extra)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/index/lookup.rb

@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-require "json"
-
-module Textus
-  class Store
-    module Index
-      class Lookup
-        def initialize(store:)
-          @store = store
-        end
-
-        def search(query, lane: nil)
-          return [] if query.to_s.strip.empty?
-
-          clauses = ["entries_fts MATCH ?"]
-          params = [query]
-          if lane
-            clauses << "entries.lane = ?"
-            params << lane
-          end
-          conditions = "WHERE #{clauses.join(" AND ")}"
-          @store.execute(
-            "SELECT entries.key, entries.lane, entries.format, entries.etag, bm25(entries_fts) AS rank
-             FROM entries_fts JOIN entries ON entries_fts.rowid = entries.rowid
-           #{conditions}
-           ORDER BY rank",
-            params,
-          )
-        rescue SQLite3::SQLException
-          []
-        end
-
-        def find_by_hash(content_hash)
-          return nil if content_hash.to_s.empty?
-
-          find_extra("content_hash", content_hash)
-        end
-
-        def find_by_url(url)
-          return nil if url.to_s.empty?
-
-          find_extra("url", url)
-        end
-
-        private
-
-        def find_extra(name, value)
-          @store.execute("SELECT key, extra FROM entries ORDER BY indexed_at DESC").each do |row|
-            extra = JSON.parse(row["extra"] || "{}")
-            return row["key"] if extra[name] == value
-          end
-          nil
-        rescue SQLite3::SQLException
-          nil
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/base.rb

@@ -1,13 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      class Base
-        def call(**)
-          raise NotImplementedError.new("#{self.class}#call")
-        end
-
-        def args = {}
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/index.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Index < Base
-        TYPE = "index"
-
-        def self.call(container:, call:) # rubocop:disable Lint/UnusedMethodArgument
-          Textus::Store::Index::Builder.new(store: container.job_store).rebuild!(resolver: container.manifest.resolver)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/materialize.rb

@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Materialize < Base
-        TYPE = "materialize"
-
-        def self.call(container:, call:, key:)
-          Textus::Produce::Engine.converge(container: container, call: call, keys: [key])
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/plan.rb

@@ -1,11 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      Plan = Data.define(:steps, :warnings) do
-        def to_h
-          { "steps" => steps, "warnings" => warnings }
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/planner.rb

@@ -1,104 +0,0 @@
-# frozen_string_literal: true
-
-module Textus
-  class Store
-    module Jobs
-      class Planner
-        ACTIONS_BY_TRIGGER = {
-          "convergence" => %w[materialize sweep index],
-          "entry.written" => %w[materialize],
-          "entry.deleted" => %w[materialize],
-          "entry.moved" => %w[materialize],
-          "proposal.accepted" => %w[materialize],
-          "proposal.rejected" => %w[materialize],
-        }.freeze
-
-        SCOPE_RESOLVERS = {
-          "materialize" => :producible_keys,
-          "sweep" => :lane_keys,
-        }.freeze
-
-        GLOBAL_ACTIONS = {
-          "index" => {},
-          "sweep" => { "scope" => {} },
-        }.freeze
-
-        def self.seed(container:, queue:, role:)
-          jobs = new(container: container).plan(
-            trigger: { "type" => "convergence" },
-            role: role,
-          )
-          jobs.each { |j| queue.enqueue(j) }
-        end
-
-        def initialize(container:)
-          @container = container
-          @manifest  = container.manifest
-        end
-
-        def plan(trigger:, role:)
-          type = trigger["type"] || trigger[:type]
-          trigger["target"] || trigger[:target]
-          return [] if type.nil?
-
-          blocks_with_react = @manifest.rules.blocks.select(&:react)
-          if blocks_with_react.any?
-            plan_from_rules(blocks_with_react, type, role)
-          else
-            plan_from_defaults(type, role)
-          end
-        end
-
-        private
-
-        def plan_from_rules(blocks, type, role)
-          jobs = []
-          blocks
-            .select { |b| matches_trigger?(b.react, type) }
-            .each do |block|
-              do_action = block.react.raw["do"]
-              Array(do_action).each do |action|
-                if (global_args = GLOBAL_ACTIONS[action])
-                  jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: global_args, role: role)
-                else
-                  resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
-                  keys = send(resolver, nil)
-                  keys.each { |key| jobs << job(action, key, role) }
-                end
-              end
-            end
-          jobs
-        end
-
-        def plan_from_defaults(type, role)
-          actions = ACTIONS_BY_TRIGGER.fetch(type, [])
-          jobs = []
-          producible_keys(nil).each { |k| jobs << job("materialize", k, role) } if actions.include?("materialize")
-          GLOBAL_ACTIONS.each do |action, args|
-            jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: args, role: role) if actions.include?(action)
-          end
-          jobs
-        end
-
-        def matches_trigger?(react, type)
-          on = react.raw["on"]
-          Array(on).include?(type)
-        end
-
-        def job(type, key, role)
-          Textus::Store::Jobs::Queue::Job.new(type: type, args: { "key" => key }, role: role)
-        end
-
-        def producible_keys(_target)
-          @manifest.data.entries
-                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
-                   .map(&:key)
-        end
-
-        def lane_keys(_target)
-          @manifest.data.entries.map(&:key)
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/queue.rb

@@ -1,154 +0,0 @@
-# frozen_string_literal: true
-
-require "digest"
-require "json"
-require "securerandom"
-require "time"
-
-module Textus
-  class Store
-    module Jobs
-      class Queue
-        VALID_STATES = %w[ready leased done failed].freeze
-
-        Leased = Data.define(:job)
-
-        class Job
-          DIGEST_BYTES = 16
-
-          attr_reader :type, :args, :role, :attempts, :max_attempts, :errors
-
-          def initialize(type:, args:, role:, attempts: 0, max_attempts: 3, errors: [])
-            @type = type.to_s
-            @args = stringify(args)
-            @role = role.to_s
-            @attempts = attempts.to_i
-            @max_attempts = max_attempts.to_i
-            @errors = Array(errors)
-          end
-
-          def id
-            "#{type}:#{Digest::SHA256.hexdigest(JSON.dump(args.sort.to_h))[0, DIGEST_BYTES]}"
-          end
-
-          private
-
-          def stringify(hash)
-            hash.to_h.transform_keys(&:to_s)
-          end
-        end
-
-        def initialize(store:)
-          @store = store
-        end
-
-        def enqueue(job)
-          now = iso_now
-          @store.execute(
-            "INSERT OR IGNORE INTO jobs (id, type, args, state, role, attempts, max_attempts, errors, lease, created_at, updated_at)
-           VALUES (?, ?, ?, 'ready', ?, ?, ?, ?, NULL, ?, ?)",
-            [job.id, job.type, JSON.dump(job.args), job.role, job.attempts, job.max_attempts, JSON.dump(job.errors), now, now],
-          )
-        end
-
-        def ready_ids
-          list(:ready)
-        end
-
-        def lease(worker_id:, lease_ttl:)
-          now = Time.now.utc
-          expires_at = now + lease_ttl
-          token = SecureRandom.hex(8)
-          marked_lease = JSON.dump({ "worker_id" => worker_id, "expires_at" => expires_at.iso8601, "token" => token })
-
-          @store.execute(
-            "UPDATE jobs
-              SET state = 'leased', lease = ?, updated_at = ?
-            WHERE id = (
-              SELECT id FROM jobs WHERE state = 'ready' ORDER BY created_at, id LIMIT 1
-            )",
-            [marked_lease, now.iso8601],
-          )
-          row = @store.execute("SELECT * FROM jobs WHERE state = 'leased' AND lease = ? LIMIT 1", [marked_lease]).first
-          return nil unless row
-
-          Leased.new(job_from_row(row))
-        end
-
-        def ack(leased)
-          @store.execute(
-            "UPDATE jobs SET state = 'done', lease = NULL, updated_at = ? WHERE id = ? AND state = 'leased'",
-            [iso_now, leased.job.id],
-          )
-        end
-
-        def fail(leased, error:)
-          job = leased.job
-          attempts = job.attempts + 1
-          errors = job.errors + [{ "attempt" => attempts, "error" => error, "at" => iso_now }]
-          dead = attempts >= job.max_attempts
-          state = dead ? "failed" : "ready"
-          @store.execute(
-            "UPDATE jobs SET state = ?, attempts = ?, errors = ?, lease = NULL, updated_at = ? WHERE id = ?",
-            [state, attempts, JSON.dump(errors), iso_now, job.id],
-          )
-          dead ? :dead_lettered : :requeued
-        end
-
-        def reclaim(now:)
-          rows = @store.execute("SELECT id, lease FROM jobs WHERE state = 'leased'")
-          expired = rows.select do |row|
-            lease = JSON.parse(row["lease"] || "{}")
-            expires_at = lease["expires_at"]
-            expires_at.nil? || Time.parse(expires_at) <= now
-          end
-          expired.each do |row|
-            @store.execute(
-              "UPDATE jobs SET state = 'ready', lease = NULL, updated_at = ? WHERE id = ?",
-              [now.utc.iso8601, row["id"]],
-            )
-          end
-          expired.size
-        end
-
-        def list(state)
-          state = state.to_s
-          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
-
-          @store.execute("SELECT id FROM jobs WHERE state = ? ORDER BY created_at, id", [state]).map { |row| row["id"] }
-        end
-
-        def retry_failed(job_id)
-          @store.execute(
-            "UPDATE jobs SET state = 'ready', attempts = 0, errors = ?, lease = NULL, updated_at = ? WHERE id = ? AND state = 'failed'",
-            [JSON.dump([]), iso_now, job_id],
-          )
-        end
-
-        def purge(state)
-          state = state.to_s
-          raise Textus::UsageError.new("unknown job state: #{state}") unless VALID_STATES.include?(state)
-
-          @store.execute("DELETE FROM jobs WHERE state = ?", [state])
-        end
-
-        private
-
-        def job_from_row(row)
-          Job.new(
-            type: row["type"],
-            args: JSON.parse(row["args"] || "{}"),
-            role: row["role"],
-            attempts: row["attempts"],
-            max_attempts: row["max_attempts"],
-            errors: JSON.parse(row["errors"] || "[]"),
-          )
-        end
-
-        def iso_now
-          Time.now.utc.iso8601
-        end
-      end
-    end
-  end
-end

data/lib/textus/store/jobs/registry.rb

@@ -1,19 +0,0 @@
-module Textus
-  class Store
-    module Jobs
-      module Registry
-        class UnknownJob < KeyError; end
-
-        JOBS = {
-          "index" => Store::Jobs::Index,
-          "materialize" => Store::Jobs::Materialize,
-          "sweep" => Store::Jobs::Sweep,
-        }.freeze
-
-        def self.fetch(type)
-          JOBS.fetch(type.to_s) { raise UnknownJob.new("Unknown job type: #{type}") }
-        end
-      end
-    end
-  end
-end