textus 0.54.2 → 0.55.2

data/lib/textus/store/container.rb

@@ -0,0 +1,173 @@
+module Textus
+  class Store
+    class Container
+      Infrastructure = Data.define(:file_store, :schemas, :audit_log, :job_store, :layout)
+      Coordination   = Data.define(:manifest, :workflows, :pipeline)
+
+      def self.attribute_names
+        @attribute_names ||= [:root] + Infrastructure.members + Coordination.members
+      end
+
+      def initialize(infra, coord)
+        @infra = infra
+        @coord = coord
+      end
+
+      attr_reader :infra, :coord, :pipeline, :reader, :writer
+
+      def root
+        @infra.layout.root
+      end
+
+      Infrastructure.members.each do |name|
+        define_method(name) { @infra.public_send(name) }
+      end
+
+      Coordination.members.each do |name|
+        define_method(name) { @coord.public_send(name) }
+      end
+
+      def wire!(pipeline:, reader:, writer:)
+        @pipeline = pipeline
+        @reader   = reader
+        @writer   = writer
+        @coord    = Coordination.new(
+          manifest: @coord.manifest,
+          workflows: @coord.workflows,
+          pipeline: pipeline,
+        )
+        self
+      end
+
+      def self.build(infra, coord_seed)
+        coord = Coordination.new(
+          manifest: coord_seed.manifest,
+          workflows: coord_seed.workflows,
+          pipeline: nil,
+        )
+        container = new(infra, coord)
+        pipeline = build_pipeline(container)
+        reader   = Textus::Store::Entry::Reader.from(container: container)
+        writer   = create_writer_factory(container)
+        container.wire!(pipeline: pipeline, reader: reader, writer: writer)
+      end
+
+      def self.orchestration_for(container)
+        Orchestration.new(
+          list_keys: Handlers::Read::ListKeys.new(manifest: container.manifest, job_store: container.job_store),
+          move_key: Handlers::Write::MoveKey.new(container: container, manifest: container.manifest),
+          delete_key: Handlers::Write::DeleteKey.new(container: container),
+          audit_entries: Handlers::Read::AuditEntries.new(manifest: container.manifest, audit_log: container.audit_log),
+        )
+      end
+
+      def self.build_pipeline(container) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        registry = Dispatch::HandlerRegistry.new
+        fe   = freshness_evaluator(container)
+        orch = orchestration_for(container)
+
+        registry.register(Dispatch::Contracts::GetEntry,
+                          Handlers::Read::GetEntry.new(container: container, freshness_evaluator: fe))
+        registry.register(Dispatch::Contracts::PutEntry,
+                          Handlers::Write::PutEntry.new(container: container))
+        registry.register(Dispatch::Contracts::ListKeys,
+                          Handlers::Read::ListKeys.new(manifest: container.manifest, job_store: container.job_store))
+        registry.register(Dispatch::Contracts::DeleteKey,
+                          Handlers::Write::DeleteKey.new(container: container))
+        registry.register(Dispatch::Contracts::MoveKey,
+                          Handlers::Write::MoveKey.new(container: container, manifest: container.manifest))
+        registry.register(Dispatch::Contracts::ProposeEntry,
+                          Handlers::Write::ProposeEntry.new(container: container))
+        registry.register(Dispatch::Contracts::AcceptProposal,
+                          Handlers::Write::AcceptProposal.new(container: container))
+        registry.register(Dispatch::Contracts::RejectProposal,
+                          Handlers::Write::RejectProposal.new(container: container))
+        registry.register(Dispatch::Contracts::EnqueueJob,
+                          Handlers::Write::EnqueueJob.new(job_store: container.job_store))
+        registry.register(Dispatch::Contracts::WhereEntry,
+                          Handlers::Read::WhereEntry.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::UidEntry,
+                          Handlers::Read::UidEntry.new(container: container))
+        registry.register(Dispatch::Contracts::DepsEntry,
+                          Handlers::Read::DepsEntry.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::RdepsEntry,
+                          Handlers::Read::RdepsEntry.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::BootStore,
+                          Handlers::Maintenance::BootStore.new(container: container))
+        registry.register(Dispatch::Contracts::DoctorStore,
+                          Handlers::Maintenance::DoctorStore.new(container: container))
+        registry.register(Dispatch::Contracts::PublishedEntries,
+                          Handlers::Maintenance::PublishedEntries.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::RuleExplain,
+                          Handlers::Maintenance::RuleExplain.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::RuleList,
+                          Handlers::Maintenance::RuleList.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::SchemaEnvelope,
+                          Handlers::Maintenance::SchemaEnvelope.new(manifest: container.manifest, schemas: container.schemas))
+        registry.register(Dispatch::Contracts::DrainStore,
+                          Handlers::Maintenance::DrainStore.new(container: container, job_store: container.job_store))
+        registry.register(Dispatch::Contracts::IngestEntry,
+                          Handlers::Maintenance::IngestEntry.new(container: container))
+        registry.register(Dispatch::Contracts::JobsAction,
+                          Handlers::Maintenance::JobsAction.new(job_store: container.job_store))
+        registry.register(Dispatch::Contracts::RuleLint,
+                          Handlers::Maintenance::RuleLint.new(manifest: container.manifest))
+        registry.register(Dispatch::Contracts::DataMv,
+                          Handlers::Write::DataMv.new(container: container))
+        registry.register(Dispatch::Contracts::AuditEntries,
+                          Handlers::Read::AuditEntries.new(manifest: container.manifest, audit_log: container.audit_log))
+        registry.register(Dispatch::Contracts::PulseEntries,
+                          Handlers::Read::PulseEntries.new(
+                            manifest: container.manifest,
+                            audit_log: container.audit_log,
+                            file_store: container.file_store,
+                            job_store: container.job_store,
+                            orchestration: orch,
+                          ))
+        registry.register(Dispatch::Contracts::BlameEntry,
+                          Handlers::Read::BlameEntry.new(manifest: container.manifest, orchestration: orch))
+        registry.register(Dispatch::Contracts::KeyMvPrefix,
+                          Handlers::Write::KeyMvPrefix.new(orchestration: orch))
+        registry.register(Dispatch::Contracts::KeyDeletePrefix,
+                          Handlers::Write::KeyDeletePrefix.new(orchestration: orch))
+
+        Dispatch::Pipeline.new(
+          registry: registry,
+          container: container,
+          middleware: [
+            Dispatch::Middleware::Binder.new,
+            Dispatch::Middleware::Auth.new,
+            Dispatch::Middleware::AuditIndex.new(
+              job_store: container.job_store,
+              audit_log: container.audit_log,
+            ),
+            Dispatch::Middleware::Cascade.new,
+          ],
+        )
+      end
+      private_class_method :build_pipeline
+
+      def self.create_writer_factory(container)
+        lambda do |call|
+          Textus::Store::Entry::Writer.new(
+            file_store: container.file_store,
+            manifest: container.manifest,
+            schemas: container.schemas,
+            audit_log: container.audit_log,
+            call: call,
+            reader: container.reader,
+            layout: container.layout,
+          )
+        end
+      end
+
+      def self.freshness_evaluator(container)
+        Store::Freshness::TtlEvaluator.new(
+          manifest: container.manifest,
+          file_stat: Textus::Port::Storage::FileStat.new,
+          clock: Textus::Port::Clock.new,
+        )
+      end
+    end
+  end
+end

data/lib/textus/store/cursor.rb

@@ -0,0 +1,26 @@
+require "fileutils"
+
+module Textus
+  class Store
+    # Per-role cursor cache under <root>/.state/cursors/<role>. A convenience so
+    # `textus pulse` (no --since) means "since I last looked". Gitignored;
+    # losing it just re-emits recent deltas, never corrupts the store. ADR 0036/0038.
+    class Cursor
+      def initialize(root:, role:)
+        @path = Store::Layout.new(root).cursor_path(role)
+      end
+
+      def read
+        Integer(File.read(@path).strip)
+      rescue Errno::ENOENT, ArgumentError
+        0
+      end
+
+      def write(seq)
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.write(@path, seq.to_s)
+        seq
+      end
+    end
+  end
+end

data/lib/textus/store/entry/reader.rb

@@ -0,0 +1,47 @@
+module Textus
+  class Store
+    module Entry
+      # Read-only counterpart to EnvelopeWriter. Resolves a key, reads the
+      # bytes, parses them via the format strategy, and hands back an
+      # Envelope. Used by Mv (pre-move inspection) and by EnvelopeWriter
+      # (existing-meta lookup for the uid/sources preservation step in #put).
+      #
+      # No audit, no events, no permission checks — those live one layer up.
+      class Reader
+        def self.from(container:)
+          # Prefer a cached reader on the container (injection point) for
+          # tests and alternative runtimes. Fall back to constructing one.
+          return container.reader if container.respond_to?(:reader) && container.reader
+
+          new(file_store: container.file_store, manifest: container.manifest,
+              layout: container.layout)
+        end
+
+        def initialize(file_store:, manifest:, layout:)
+          @file_store = file_store
+          @manifest   = manifest
+          @layout     = layout
+        end
+
+        def read(key)
+          res = @manifest.resolver.resolve(key)
+          path = res.path
+          return nil unless @file_store.exists?(path)
+
+          mentry = res.entry
+          raw = @file_store.read(path)
+          parsed = Format.for(mentry.format).parse(raw, path: path)
+          Textus::Value::Envelope.build(
+            key: key, mentry: mentry, path: path,
+            meta: parsed["_meta"], body: parsed["body"],
+            etag: Value::Etag.for_bytes(raw), content: parsed["content"]
+          )
+        end
+
+        def exists?(key)
+          @file_store.exists?(@manifest.resolver.resolve(key).path)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/entry/writer.rb

@@ -0,0 +1,219 @@
+require "fileutils"
+
+module Textus
+  class Store
+    module Entry
+      # Owns the write pipeline (validate, serialize, etag-check, write, audit).
+      # Talks to ports (FileStore, Schemas, AuditLog, Manifest) and an
+      # Reader for the existing-uid lookup.
+      #
+      # Invariant: every public method's final action is @audit_log.append(...).
+      #
+      # No permission check, no event firing — those belong to the caller
+      # (Write::Put / ::Delete / ::Mv).
+      class Writer
+        def self.from(container:, call:)
+          # If the container exposes a writer factory (in tests we set this),
+          # use it. Otherwise, construct a fresh Writer.
+          return container.writer.call(call) if container.respond_to?(:writer) && container.writer
+
+          new(
+            file_store: container.file_store, manifest: container.manifest,
+            schemas: container.schemas, audit_log: container.audit_log,
+            call: call, reader: Reader.from(container: container),
+            layout: container.layout
+          )
+        end
+
+        def initialize(file_store:, manifest:, schemas:, audit_log:, call:, reader:, layout:)
+          @file_store = file_store
+          @manifest   = manifest
+          @schemas    = schemas
+          @audit_log  = audit_log
+          @call       = call
+          @reader     = reader
+          @layout     = layout
+        end
+
+        def put(key, mentry:, payload:, if_etag: nil)
+          path  = resolve_path(key)
+          meta  = payload.meta || {}
+          content = payload.content
+
+          existing_env   = read_existing(key)
+          existing_meta  = existing_env ? existing_env.meta : {}
+          meta, content  = inject_meta(meta, content, existing_meta, mentry.format)
+
+          bytes, eff_meta, eff_body, eff_content = serialize_entry(mentry, path, meta, payload, content)
+
+          enforce_name_match!(path, eff_meta, mentry.format)
+          validate_schema(mentry, eff_meta, eff_content)
+          validate_raw(eff_meta, eff_content, mentry.lane, mentry.format)
+
+          etag_before = check_etag!(path, key, if_etag)
+          write_bytes(path, bytes)
+
+          envelope = build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes)
+          audit_put(key, etag_before, envelope.etag)
+          envelope
+        end
+
+        def delete(key, mentry: nil, if_etag: nil) # rubocop:disable Lint/UnusedMethodArgument
+          # `mentry:` is accepted for symmetry with `put` / `move` and to
+          # leave room for future format-specific delete hooks; no field
+          # on it is needed today.
+          path = @manifest.resolver.resolve(key).path
+          raise UnknownKey.new(key, suggestions: @manifest.resolver.suggestions_for(key)) unless @file_store.exists?(path)
+
+          etag_before = @file_store.etag(path)
+          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && if_etag != etag_before
+
+          @file_store.delete(path)
+          prune_empty_parents(path)
+          @audit_log.append(
+            role: @call.role, verb: "key_delete", key: key,
+            etag_before: etag_before, etag_after: nil,
+            extras: @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
+          )
+        end
+
+        def move(from_key:, to_key:, new_mentry:, if_etag: nil)
+          from_path = @manifest.resolver.resolve(from_key).path
+          to_path   = @manifest.resolver.resolve(to_key).path
+          raise UnknownKey.new(from_key, suggestions: @manifest.resolver.suggestions_for(from_key)) unless @file_store.exists?(from_path)
+
+          etag_before = @file_store.etag(from_path)
+          raise EtagMismatch.new(from_key, if_etag, etag_before) if if_etag && if_etag != etag_before
+
+          @file_store.mv(from_path, to_path)
+          prune_empty_parents(from_path)
+          basename = to_key.split(".").last
+          Format.for(new_mentry.format).rewrite_name(to_path, basename)
+          etag_after = Value::Etag.for_file(to_path)
+
+          envelope = @reader.read(to_key)
+
+          extras = {
+            "from_key" => from_key, "to_key" => to_key,
+            "from_path" => from_path, "to_path" => to_path,
+            "uid" => envelope.uid
+          }
+          extras["correlation_id"] = @call.correlation_id if @call.correlation_id
+
+          @audit_log.append(
+            role: @call.role, verb: "key_mv", key: to_key,
+            etag_before: etag_before, etag_after: etag_after,
+            extras: extras
+          )
+
+          envelope
+        end
+
+        private
+
+        # After a file leaves a directory (delete or move-source), remove any
+        # now-empty parent dirs so bulk move/delete doesn't accrue orphan dirs
+        # (F3 of #161). Floored at the entry's *zone directory* — a zone is a
+        # declared, first-class container, so its own dir is preserved even when
+        # momentarily empty; only the sub-dirs the bulk op carved out are
+        # pruned. Stops at the first non-empty ancestor, so a dir holding a
+        # `.gitkeep` or sibling entries survives. Best-effort: a lost race or a
+        # non-empty dir is silently fine, never fatal to the write.
+        def prune_empty_parents(path)
+          floor = @layout.lane_floor(path)
+          return unless floor
+
+          dir = File.dirname(path)
+          while dir.start_with?("#{floor}/") && @file_store.dir_empty?(dir)
+            @file_store.rmdir(dir)
+            dir = File.dirname(dir)
+          end
+        rescue SystemCallError
+          nil
+        end
+
+        def read_existing(key)
+          @reader.read(key)
+        end
+
+        def inject_meta(meta, content, existing_meta, format)
+          Envelope::Meta.inject_all(
+            meta, content, existing_meta,
+            format: format,
+            etag_for: method(:resolve_source_etag)
+          )
+        end
+
+        def resolve_source_etag(key)
+          path = @manifest.resolver.resolve(key).path
+          return nil unless @file_store.exists?(path)
+
+          Value::Etag.for_file(path)
+        rescue Textus::Error
+          nil
+        end
+
+        def resolve_path(key)
+          @manifest.resolver.resolve(key).path
+        end
+
+        def serialize_entry(mentry, path, meta, payload, content)
+          Textus::Format.for(mentry.format).serialize_for_put(
+            meta: meta, body: payload.body, content: content, path: path,
+          )
+        end
+
+        def enforce_name_match!(path, meta, format)
+          Textus::Format.for(format).enforce_name_match!(path, meta)
+        end
+
+        def validate_schema(mentry, eff_meta, eff_content)
+          schema = @schemas.fetch_or_nil(mentry.schema)
+          return unless schema
+
+          Format.for(mentry.format).validate_against(
+            schema,
+            { "_meta" => eff_meta, "content" => eff_content },
+          )
+        end
+
+        def validate_raw(eff_meta, eff_content, lane, format)
+          Textus::Format.for(format).validate_raw_entry!(
+            { "_meta" => eff_meta, "content" => eff_content },
+            lane,
+          )
+        end
+
+        def check_etag!(path, key, if_etag)
+          etag_before = @file_store.exists?(path) ? @file_store.etag(path) : nil
+          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
+
+          etag_before
+        end
+
+        def write_bytes(path, bytes)
+          @file_store.write(path, bytes)
+        end
+
+        def build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes = nil)
+          raw = bytes || @file_store.read(path)
+          Textus::Value::Envelope.build(
+            key: key, mentry: mentry, path: path,
+            meta: eff_meta, body: eff_body,
+            etag: Value::Etag.for_bytes(raw),
+            content: eff_content
+          )
+        end
+
+        def audit_put(key, etag_before, etag_after)
+          extras = @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
+          @audit_log.append(
+            role: @call.role, verb: "put", key: key,
+            etag_before: etag_before, etag_after: etag_after,
+            extras: extras
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/envelope/meta.rb

@@ -0,0 +1,61 @@
+require "securerandom"
+
+module Textus
+  class Store
+    module Envelope
+      module Meta
+        NO_META_FORMATS = %w[text].freeze
+
+        FIELDS = {
+          "uid" => {
+            inject: lambda { |meta, content, existing_meta, **_opts|
+              m = meta.is_a?(Hash) ? meta.dup : {}
+              existing = existing_meta.is_a?(Hash) ? existing_meta["uid"] : nil
+              m["uid"] = existing || Textus::Value::Uid.mint unless m["uid"].is_a?(String) && !m["uid"].empty?
+              [m, content]
+            },
+          },
+          "sources" => {
+            inject: lambda { |meta, content, existing_meta, etag_for: nil|
+              m = meta.is_a?(Hash) ? meta.dup : {}
+              existing = existing_meta.is_a?(Hash) ? existing_meta["sources"] : nil
+
+              if m.key?("sources")
+                raise Textus::BadContent.new(nil, "_meta.sources must be an array") unless m["sources"].is_a?(Array)
+
+                m["sources"] = m["sources"].map { |s| Meta.normalize_source!(s, etag_for) }
+              elsif existing.is_a?(Array) && !existing.empty?
+                m["sources"] = existing
+              end
+
+              [m, content]
+            },
+          },
+        }.freeze
+
+        def self.inject_all(meta, content, existing_meta = {}, format: nil, etag_for: nil)
+          return [meta, content] if NO_META_FORMATS.include?(format)
+
+          FIELDS.each_value do |field|
+            meta, content = field[:inject].call(meta, content, existing_meta, etag_for: etag_for)
+          end
+
+          [meta, content]
+        end
+
+        def self.normalize_source!(src, etag_for)
+          key = case src
+                when String then src
+                when Hash   then src["key"]
+                end
+
+          raise Textus::BadContent.new(nil, "each source must be a string key or { key: } object") unless key.is_a?(String)
+          raise Textus::BadContent.new(nil, "each source key must be a non-empty string") if key.empty?
+
+          etag = etag_for&.call(key)
+          etag ? { "key" => key, "etag" => etag } : { "key" => key }
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/freshness/drift_detector.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Textus
+  class Store
+    module Freshness
+      class DriftDetector
+        def initialize(manifest:, file_stat:, clock:)
+          @manifest  = manifest
+          @file_stat = file_stat
+          @clock     = clock
+        end
+
+        def drift_rows(mentry)
+          return [] unless mentry.external?
+
+          path   = Textus::Key::Path.resolve(@manifest.data, mentry)
+          reason = drift_reason(mentry, path)
+          reason ? [drift_row(mentry, path, reason)] : []
+        end
+
+        private
+
+        def drift_reason(mentry, path)
+          return "derived entry has never been generated" unless @file_stat.exists?(path)
+
+          generated_at = generated_at_of(mentry, path)
+          return "missing generated.at frontmatter" unless generated_at
+
+          gen_time = parse_time(generated_at)
+          return "unparseable generated.at: #{generated_at.inspect}" unless gen_time
+
+          offender = newest_source_after(mentry.source, gen_time)
+          "source '#{offender}' modified after generated.at" if offender
+        end
+
+        def generated_at_of(mentry, path)
+          Textus::Format.for(mentry.format).parse(@file_stat.read(path), path: path)["_meta"]
+                        .dig("generated", "at")
+        end
+
+        def parse_time(str)
+          Time.parse(str.to_s)
+        rescue StandardError
+          nil
+        end
+
+        def newest_source_after(external_src, gen_time)
+          Array(external_src.sources).each do |src|
+            offender = check_source(src, gen_time)
+            return offender if offender
+          end
+          nil
+        end
+
+        def check_source(src, gen_time)
+          if src.match?(/\A[a-z0-9.][a-z0-9._-]*\z/) && !src.include?("/")
+            @manifest.resolver.enumerate(prefix: src).each do |row|
+              return src if @file_stat.mtime(row[:path]) > gen_time
+            end
+            nil
+          else
+            check_filesystem_source(src, gen_time)
+          end
+        end
+
+        def check_filesystem_source(src, gen_time)
+          abs = absolutize_source(src)
+          if @file_stat.directory?(abs)
+            dir_has_newer_file?(abs, gen_time) ? src : nil
+          elsif @file_stat.exists?(abs) && @file_stat.mtime(abs) > gen_time
+            src
+          end
+        end
+
+        def absolutize_source(src)
+          File.absolute_path?(src) ? src : File.join(File.dirname(@manifest.data.root), src)
+        end
+
+        def dir_has_newer_file?(abs, gen_time)
+          @file_stat.glob(File.join(abs, "**", "*")).any? do |fpath|
+            !@file_stat.directory?(fpath) && @file_stat.exists?(fpath) && @file_stat.mtime(fpath) > gen_time
+          end
+        end
+
+        def drift_row(mentry, path, reason)
+          { "key" => mentry.key, "path" => path, "generator" => mentry.source.command, "reason" => reason }
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/freshness/evaluator.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Freshness
+      # Thin facade delegating to the focused TtlEvaluator and DriftDetector.
+      # Prefer using TtlEvaluator or DriftDetector directly.
+      class Evaluator
+        def initialize(manifest:, file_stat:, clock:)
+          @ttl   = TtlEvaluator.new(manifest: manifest, file_stat: file_stat, clock: clock)
+          @drift = DriftDetector.new(manifest: manifest, file_stat: file_stat, clock: clock)
+        end
+
+        def verdict(mentry) = @ttl.verdict(mentry)
+        def stale_keys(**) = @ttl.stale_keys(**)
+        def drift_rows(mentry) = @drift.drift_rows(mentry)
+      end
+    end
+  end
+end