textus 0.55.1 → 0.55.2

data/lib/textus/handlers/maintenance/ingest_entry.rb

@@ -0,0 +1,159 @@
+require "fileutils"
+require "date"
+require "digest"
+
+module Textus
+  module Handlers
+    module Maintenance
+      class IngestEntry
+        SOURCE_KINDS = %w[url file asset].freeze
+        CONTENT_HASH_ALGO = "sha256"
+
+        def initialize(container:)
+          @container = container
+        end
+
+        def call(command, call)
+          unless SOURCE_KINDS.include?(command.kind)
+            return Value::Result.failure(:usage_error,
+                                         "ingest kind must be one of #{SOURCE_KINDS.join("|")}")
+          end
+
+          case command.kind
+          when "url"   then return Value::Result.failure(:usage_error, "ingest url requires url") unless command.url
+          when "file"  then return Value::Result.failure(:usage_error, "ingest file requires path") unless command.path
+          when "asset"
+            return Value::Result.failure(:usage_error, "ingest asset requires path") unless command.path
+            return Value::Result.failure(:usage_error, "ingest asset requires lane") unless command.lane
+          end
+
+          now = Time.now.utc
+          key = derive_key(now, command.kind, command.slug)
+          content_hash = compute_content_hash(command)
+          mentry = @container.manifest.resolver.resolve(key).entry
+          ts = now.iso8601
+
+          structured = build_structured(ts, now, content_hash, command)
+          store = @container.job_store
+          index = Textus::Store::Index::Lookup.new(store:)
+
+          duplicate_key = find_duplicate(index, content_hash, command)
+
+          env = if duplicate_key && duplicate_key != key
+                  supersede_entry(duplicate_key, key, structured, call, store, command)
+                else
+                  write_entry(key, structured, mentry, call)
+                end
+
+          rebuild_index(store)
+          Value::Result.success(env)
+        end
+
+        private
+
+        def derive_key(now, kind, slug)
+          date = now.strftime("%Y.%m.%d")
+          "raw.#{date}.#{kind}-#{slug}"
+        end
+
+        def compute_content_hash(command)
+          digest = Digest::SHA256.new
+          case command.kind
+          when "url" then digest.update(command.url)
+          when "file", "asset" then digest.file(command.path)
+          end
+          "#{CONTENT_HASH_ALGO}:#{digest.hexdigest}"
+        end
+
+        def build_structured(timestamp, now, content_hash, command)
+          base = { "ingested_at" => timestamp, "content_hash" => content_hash }
+          case command.kind
+          when "url"
+            base.merge("source" => { "kind" => "url", "url" => command.url,
+                                     "label" => command.label || command.url }, "body" => nil)
+          when "file"
+            base.merge("source" => { "kind" => "file", "path" => command.path,
+                                     "label" => command.label || File.basename(command.path) },
+                       "body" => File.read(command.path))
+          when "asset"
+            asset_rel = copy_asset(now, command.path, command.lane)
+            base.merge("source" => { "kind" => "asset",
+                                     "label" => command.label || File.basename(command.path) },
+                       "asset" => asset_rel, "body" => nil)
+          end
+        end
+
+        def copy_asset(now, path, lane)
+          date_path = now.strftime("%Y/%m/%d")
+          filename  = File.basename(path)
+          assets_dir = @container.layout.asset_raw_dir(date_path, lane)
+          FileUtils.mkdir_p(assets_dir)
+          FileUtils.cp(path, File.join(assets_dir, filename))
+          sentinel = @container.layout.asset_sentinel_path
+          File.write(sentinel, "*\n") unless File.exist?(sentinel)
+          "raw/#{date_path}/#{lane}/#{filename}"
+        end
+
+        def write_entry(key, structured, mentry, call)
+          writer = Store::Entry::Writer.from(container: @container, call: call)
+          writer.put(key, mentry: mentry,
+                          payload: Textus::Value::Payload.new(meta: nil, body: nil, content: structured))
+        end
+
+        def find_duplicate(index, content_hash, command)
+          dup = index.find_by_hash(content_hash)
+          return dup if dup
+          return unless command.kind == "url"
+
+          index.find_by_url(command.url)
+        end
+
+        def supersede_entry(old_key, new_key, structured, call, store, command)
+          old_mentry = @container.manifest.resolver.resolve(old_key).entry
+          reader = Store::Entry::Reader.from(container: @container)
+          old_env = reader.read(old_key)
+          old_content = old_env&.content || {}
+          tombstone = {}
+          %w[ingested_at].each { |k| tombstone[k] = old_content[k] if old_content.key?(k) }
+          source_kind = old_content.dig("source", "kind")
+          tombstone["source"] = { "kind" => source_kind } if source_kind
+          tombstone["superseded_by"] = new_key
+
+          writer = Store::Entry::Writer.from(container: @container, call: call)
+          writer.put(old_key, mentry: old_mentry,
+                              payload: Textus::Value::Payload.new(meta: nil, body: nil, content: tombstone))
+
+          structured["supersedes"] = old_key
+          env = write_entry(new_key, structured,
+                            @container.manifest.resolver.resolve(new_key).entry, call)
+
+          move_asset(old_content["asset"], command.lane) if command.kind == "asset" && old_content["asset"]
+
+          rebuild_index(store)
+          env
+        end
+
+        def move_asset(old_rel, lane)
+          old_path = @container.layout.asset_resolve(old_rel)
+          return unless File.exist?(old_path)
+
+          now = Time.now.utc
+          date_path = now.strftime("%Y/%m/%d")
+          filename = File.basename(old_path)
+          new_dir = @container.layout.asset_raw_dir(date_path, lane)
+          new_path = File.join(new_dir, filename)
+          return if old_path == new_path
+
+          FileUtils.mkdir_p(new_dir)
+          FileUtils.mv(old_path, new_path)
+        rescue Errno::ENOENT, Errno::EACCES => e
+          warn "[textus ingest] could not move asset #{old_rel}: #{e.message}"
+        end
+
+        def rebuild_index(store)
+          Textus::Store::Index::Builder.new(store:).rebuild!(resolver: @container.manifest.resolver)
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/jobs_action.rb

@@ -0,0 +1,21 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class JobsAction
+        def initialize(job_store:)
+          @job_store = job_store
+        end
+
+        def call(command, _call)
+          queue = Textus::Store::Jobs::Queue.new(store: @job_store)
+          case command.action
+          when "retry" then queue.retry_failed(command.job_id)
+          when "purge" then queue.purge(command.state)
+          end
+          Value::Result.success("protocol" => Textus::PROTOCOL, "ok" => true,
+                                "state" => command.state, "jobs" => queue.list(command.state))
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/published_entries.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class PublishedEntries
+        def initialize(manifest:)
+          @manifest = manifest
+        end
+
+        def call(_command, _call)
+          Value::Result.success(@manifest.data.entries.reject { |entry| entry.publish_to.empty? }.map do |entry|
+            { "key" => entry.key, "publish_to" => entry.publish_to }
+          end)
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/rule_explain.rb

@@ -0,0 +1,77 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class RuleExplain
+        def initialize(manifest:)
+          @manifest = manifest
+        end
+
+        def call(command, _call)
+          key = command.key
+          result = if command.detail
+                     explain(key)
+                   else
+                     effective(key)
+                   end
+          Value::Result.success(result)
+        end
+
+        LEAN_FIELDS = Textus::Manifest::Schema::FIELD_REGISTRY
+                      .select { |_, m| m[:in_rule_explain].include?(:lean) }.keys.freeze
+        DETAIL_FIELDS = Textus::Manifest::Schema::FIELD_REGISTRY
+                        .select { |_, m| m[:in_rule_explain].include?(:detail) }.keys.freeze
+        EFFECTIVE_FIELDS = DETAIL_FIELDS.select { |f| Textus::Manifest::Schema::FIELD_REGISTRY[f][:policy_class] }.freeze
+
+        private
+
+        def effective(key)
+          set = @manifest.rules.for(key)
+          LEAN_FIELDS.each_with_object({}) do |field, out|
+            value = set.public_send(field)
+            out[field.to_s] = lean_value(field, value) unless value.nil?
+          end
+        end
+
+        def lean_value(field, value)
+          case field
+          when :retention then retention_hash(value, string_keys: true)
+          when :react then value.to_h
+          else value
+          end
+        end
+
+        def explain(key)
+          matching = @manifest.rules.explain(key)
+          winners = @manifest.rules.for(key)
+          {
+            key: key,
+            matched_blocks: matching.map do |block|
+              { match: block.match }.merge(DETAIL_FIELDS.to_h { |f| [f, !block.public_send(f).nil?] })
+            end,
+            effective: EFFECTIVE_FIELDS.to_h { |f| [f, effective_value(f, winners.public_send(f))] },
+            guards: Textus::Manifest::Policy::Predicates::FLOOR.keys.to_h do |action|
+              floor = Textus::Manifest::Policy::Predicates::FLOOR.fetch(action, [])
+              rule = Array(@manifest.rules.for(key).guard&.dig(action.to_s))
+              [action, { floor: floor, rule: rule }]
+            end,
+          }
+        end
+
+        def effective_value(field, value)
+          return nil if value.nil?
+
+          case field
+          when :retention then retention_hash(value, string_keys: false)
+          when :react then value.to_h
+          else value
+          end
+        end
+
+        def retention_hash(retention, string_keys:)
+          h = { ttl_seconds: retention.ttl_seconds, action: retention.action }
+          string_keys ? h.transform_keys(&:to_s) : h
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/rule_lint.rb

@@ -0,0 +1,54 @@
+require "yaml"
+
+module Textus
+  module Handlers
+    module Maintenance
+      class RuleLint
+        def initialize(manifest:)
+          @manifest = manifest
+        end
+
+        def call(command, _call)
+          root = @manifest.data.root
+          live_rules = current_rules(root)
+          candidate_result = parse_candidate(command.candidate_yaml)
+          return candidate_result if candidate_result.is_a?(Value::Result) && candidate_result.failure?
+
+          candidate_rules = candidate_result
+          live_by_match = live_rules.to_h { |rule| [rule["match"], rule] }
+          candidate_by_match = candidate_rules.to_h { |rule| [rule["match"], rule] }
+
+          steps = (candidate_by_match.keys - live_by_match.keys).map do |match|
+            { "op" => "add_rule", "match" => match, "rule" => candidate_by_match[match] }
+          end
+          (live_by_match.keys - candidate_by_match.keys).each do |match|
+            steps << { "op" => "remove_rule", "match" => match }
+          end
+          (live_by_match.keys & candidate_by_match.keys).each do |match|
+            next if live_by_match[match] == candidate_by_match[match]
+
+            steps << { "op" => "change_rule", "match" => match, "from" => live_by_match[match], "to" => candidate_by_match[match] }
+          end
+
+          Value::Result.success(Textus::Store::Jobs::Plan.new(steps: steps, warnings: []))
+        end
+
+        private
+
+        def current_rules(root)
+          raw = YAML.safe_load_file(File.join(root, "manifest.yaml"), permitted_classes: [Symbol], aliases: false)
+          Array(raw["rules"])
+        end
+
+        def parse_candidate(yaml_text)
+          raw = YAML.safe_load(yaml_text, permitted_classes: [Symbol], aliases: false)
+          return Value::Result.failure(:usage_error, "candidate is not a YAML mapping") unless raw.is_a?(Hash)
+
+          Array(raw["rules"])
+        rescue Psych::Exception => e
+          Value::Result.failure(:usage_error, "candidate YAML parse error: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/rule_list.rb

@@ -0,0 +1,32 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class RuleList
+        def initialize(manifest:)
+          @manifest = manifest
+        end
+
+        def call(_command, _call)
+          Value::Result.success(@manifest.rules.blocks.map do |block|
+            row = { "match" => block.match }
+            LIST_FIELDS.each do |field|
+              value = block.public_send(field)
+              row[field.to_s] = serialize(field, value) unless value.nil?
+            end
+            row
+          end)
+        end
+
+        LIST_FIELDS = Textus::Manifest::Schema::FIELD_REGISTRY.select { |_, m| m[:in_rule_list] }.keys.freeze
+
+        def serialize(field, value)
+          case field
+          when :retention then { "ttl_seconds" => value.ttl_seconds, "action" => value.action.to_s }
+          when :react then value.to_h
+          else value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/schema_envelope.rb

@@ -0,0 +1,19 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class SchemaEnvelope
+        def initialize(manifest:, schemas:)
+          @manifest = manifest
+          @schemas = schemas
+        end
+
+        def call(command, _call)
+          mentry = @manifest.resolver.resolve(command.key).entry
+          schema = @schemas.fetch_or_nil(mentry.schema)
+          Value::Result.success("protocol" => Textus::PROTOCOL, "key" => command.key,
+                                "schema_ref" => mentry.schema, "schema" => schema&.to_h)
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/read/audit_entries.rb

@@ -0,0 +1,48 @@
+module Textus
+  module Handlers
+    module Read
+      class AuditEntries
+        def initialize(manifest:, audit_log:)
+          @manifest = manifest
+          @audit_log = audit_log
+        end
+
+        def call(command, _call)
+          cursor_check = check_cursor_expiry(command.seq_since)
+          return cursor_check if cursor_check
+
+          rows = @audit_log.scan(
+            seq_since: command.seq_since,
+            key: command.key, role: command.role, verb: command.verb,
+            correlation_id: command.correlation_id, limit: command.limit
+          ).select do |row|
+            next false if command.lane && !key_in_lane?(row["key"], command.lane)
+            next false if command.since && (row["ts"].nil? || Time.parse(row["ts"]) < command.since)
+
+            true
+          end
+          Value::Result.success(rows)
+        end
+
+        private
+
+        def check_cursor_expiry(seq_since)
+          return unless seq_since
+
+          min = @audit_log.min_available_seq
+          return unless min && seq_since < min - 1
+
+          Value::Result.failure(:cursor_expired, "requested seq #{seq_since} is below minimum available #{min}",
+                                details: { requested: seq_since, min_available: min })
+        end
+
+        def key_in_lane?(key, lane)
+          mentry = @manifest.resolver.resolve(key).entry
+          mentry && mentry.lane == lane
+        rescue Textus::Error
+          false
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/read/blame_entry.rb

@@ -0,0 +1,71 @@
+module Textus
+  module Handlers
+    module Read
+      class BlameEntry
+        def initialize(manifest:, orchestration:)
+          @manifest = manifest
+          @orchestration = orchestration
+        end
+
+        def call(command, call)
+          root = @manifest.data.root
+          audit = @orchestration.audit_entries(key: command.key, limit: command.limit, call: call)
+          return audit if audit.failure?
+
+          audit_rows = audit.value.fetch("rows")
+
+          path = resolve_path(command.key)
+          return Value::Result.success(audit_rows.map { |row| row.merge("git" => nil) }) unless git_tracked?(path, root: root)
+
+          Value::Result.success(audit_rows.map { |row| row.merge("git" => git_commit_at(path, timestamp: row["ts"], root: root)) })
+        end
+
+        private
+
+        def resolve_path(key)
+          res = @manifest.resolver.resolve(key)
+          path = res.path
+          path || Textus::Key::Path.resolve(@manifest.data, res.entry)
+        rescue Textus::Error
+          nil
+        end
+
+        def git_tracked?(path, root:)
+          return false if path.nil? || !File.exist?(path) || !git_repo?(root)
+
+          _out, _err, status = Open3.capture3("git", "ls-files", "--error-unmatch", path, chdir: root)
+          status.success?
+        rescue Errno::ENOENT
+          false
+        end
+
+        def git_repo?(root)
+          dir = root
+          loop do
+            return true if File.directory?(File.join(dir, ".git"))
+
+            parent = File.dirname(dir)
+            return false if parent == dir
+
+            dir = parent
+          end
+        end
+
+        def git_commit_at(path, timestamp:, root:)
+          args = ["git", "log", "-1"]
+          args << "--before=#{timestamp}" if timestamp
+          args += ["--format=%H%x09%an%x09%aI%x09%s", "--", path]
+          out, _err, status = Open3.capture3(*args, chdir: root)
+          return nil unless status.success?
+
+          sha, author, date, subject = out.strip.split("\t", 4)
+          return nil if sha.nil? || sha.empty?
+
+          { "sha" => sha, "author" => author, "date" => date, "subject" => subject }
+        rescue Errno::ENOENT
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/read/deps_entry.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Handlers
+    module Read
+      class DepsEntry
+        def initialize(manifest:)
+          @manifest = manifest
+        end
+
+        def call(command, _call)
+          entry = @manifest.data.entries.find { |e| e.key == command.key }
+          deps = entry&.external? ? Array(entry.source&.sources).compact : []
+          Value::Result.success("key" => command.key, "deps" => deps.uniq)
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/read/get_entry.rb

@@ -0,0 +1,68 @@
+module Textus
+  module Handlers
+    module Read
+      class GetEntry
+        def initialize(container:, freshness_evaluator:)
+          @container = container
+          @freshness_evaluator = freshness_evaluator
+        end
+
+        def call(command, _call)
+          envelope = Store::Entry::Reader.from(container: @container).read(command.key)
+          return Value::Result.failure(:not_found, "no entry at #{command.key}") unless envelope
+
+          envelope = expand_sources(envelope, depth: 0)
+          Value::Result.success(envelope.with(freshness: @freshness_evaluator.verdict(resolve_entry(command.key))))
+        end
+
+        MAX_SOURCE_DEPTH = 5
+
+        private
+
+        def expand_sources(envelope, depth:)
+          return envelope if depth >= MAX_SOURCE_DEPTH
+
+          raw_sources = Array(envelope.meta["sources"])
+          return envelope if raw_sources.empty?
+
+          expanded = raw_sources.map { |src| expand_one_source(src, depth: depth) }
+          envelope.with(sources: expanded)
+        end
+
+        def expand_one_source(src, depth:)
+          src = { "key" => src } if src.is_a?(String)
+          return src unless src.is_a?(Hash) && src["key"].is_a?(String)
+
+          key = src["key"]
+          stored_etag = src["etag"]
+          current_etag = resolve_current_etag(key)
+          suspended = stored_etag && current_etag ? stored_etag != current_etag : false
+
+          result = src.merge("suspended" => suspended)
+
+          child_env = @container.reader.read(key)
+          if child_env
+            child_expanded = expand_sources(child_env, depth: depth + 1)
+            child_sources = Array(child_expanded.sources)
+            result = result.merge("sources" => child_sources) unless child_sources.empty?
+          end
+
+          result
+        end
+
+        def resolve_current_etag(key)
+          path = @container.manifest.resolver.resolve(key).path
+          return nil unless @container.file_store.exists?(path)
+
+          @container.file_store.etag(path)
+        rescue Textus::Error
+          nil
+        end
+
+        def resolve_entry(key)
+          @container.manifest.resolver.resolve(key).entry
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/read/list_keys.rb

@@ -0,0 +1,36 @@
+module Textus
+  module Handlers
+    module Read
+      class ListKeys
+        def initialize(manifest:, job_store: nil)
+          @manifest  = manifest
+          @job_store = job_store
+        end
+
+        def call(command, _call)
+          q      = command.respond_to?(:q)      ? command.q      : nil
+          schema = command.respond_to?(:schema) ? command.schema : nil
+
+          return sqlite_list(q: q, schema: schema, lane: command.lane, prefix: command.prefix) if @job_store && (q || schema)
+
+          manifest_list(prefix: command.prefix, lane: command.lane)
+        end
+
+        private
+
+        def sqlite_list(q:, schema:, lane:, prefix:) # rubocop:disable Naming/MethodParameterName
+          rows = @job_store.search_entries(q: q, schema: schema, lane: lane, prefix: prefix)
+          Value::Result.success((rows || []).map { |r| { "key" => r["key"], "lane" => r["lane"] } })
+        end
+
+        def manifest_list(prefix:, lane:)
+          rows = @manifest.resolver.enumerate(prefix: prefix)
+          rows = rows.select { |row| row[:manifest_entry].lane == lane } if lane
+          Value::Result.success(rows.map do |row|
+            { "key" => row[:key], "lane" => row[:manifest_entry].lane, "path" => row[:path] }
+          end)
+        end
+      end
+    end
+  end
+end