textus 0.54.2 → 0.55.1

data/lib/textus/store/envelope/writer.rb

@@ -0,0 +1,195 @@
+require "fileutils"
+
+module Textus
+  class Store
+    module Envelope
+      # Owns the write pipeline (validate, serialize, etag-check, write, audit).
+      # Talks to ports (FileStore, Schemas, AuditLog, Manifest) and an
+      # Reader for the existing-uid lookup.
+      #
+      # Invariant: every public method's final action is @audit_log.append(...).
+      #
+      # No permission check, no event firing — those belong to the caller
+      # (Write::Put / ::Delete / ::Mv).
+      class Writer
+        Payload = Data.define(:meta, :body, :content)
+
+        def self.from(container:, call:)
+          new(
+            file_store: container.file_store, manifest: container.manifest,
+            schemas: container.schemas, audit_log: container.audit_log,
+            call: call, reader: Reader.from(container: container),
+            geometry: container.geometry
+          )
+        end
+
+        def initialize(file_store:, manifest:, schemas:, audit_log:, call:, reader:, geometry:) # rubocop:disable Metrics/ParameterLists
+          @file_store = file_store
+          @manifest   = manifest
+          @schemas    = schemas
+          @audit_log  = audit_log
+          @call       = call
+          @reader     = reader
+          @geometry   = geometry
+        end
+
+        def put(key, mentry:, payload:, if_etag: nil)
+          path = resolve_path(key)
+          meta = payload.meta || {}
+          content = payload.content
+          existing_env = @reader.read(key)
+          existing_meta = existing_env ? existing_env.meta : {}
+          meta, content = Textus::Meta.inject_all(meta, content, existing_meta, format: mentry.format)
+          bytes, eff_meta, eff_body, eff_content = serialize_entry(mentry, path, meta, payload, content)
+          enforce_name_match!(path, eff_meta, mentry.format)
+          validate_schema(mentry, eff_meta, eff_content)
+          Textus::Format::Yaml.validate_raw_entry!(
+            { "_meta" => eff_meta, "content" => eff_content },
+            mentry.lane,
+          )
+          etag_before = check_etag!(path, key, if_etag)
+          write_bytes(path, bytes)
+          envelope = build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes)
+          audit_put(key, etag_before, envelope.etag)
+          envelope
+        end
+
+        def delete(key, mentry: nil, if_etag: nil) # rubocop:disable Lint/UnusedMethodArgument
+          # `mentry:` is accepted for symmetry with `put` / `move` and to
+          # leave room for future format-specific delete hooks; no field
+          # on it is needed today.
+          path = @manifest.resolver.resolve(key).path
+          raise UnknownKey.new(key, suggestions: @manifest.resolver.suggestions_for(key)) unless @file_store.exists?(path)
+
+          etag_before = @file_store.etag(path)
+          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && if_etag != etag_before
+
+          @file_store.delete(path)
+          prune_empty_parents(path)
+          @audit_log.append(
+            role: @call.role, verb: "key_delete", key: key,
+            etag_before: etag_before, etag_after: nil,
+            extras: @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
+          )
+        end
+
+        def move(from_key:, to_key:, new_mentry:, if_etag: nil)
+          from_path = @manifest.resolver.resolve(from_key).path
+          to_path   = @manifest.resolver.resolve(to_key).path
+          raise UnknownKey.new(from_key, suggestions: @manifest.resolver.suggestions_for(from_key)) unless @file_store.exists?(from_path)
+
+          etag_before = @file_store.etag(from_path)
+          raise EtagMismatch.new(from_key, if_etag, etag_before) if if_etag && if_etag != etag_before
+
+          FileUtils.mkdir_p(File.dirname(to_path))
+          FileUtils.mv(from_path, to_path)
+          prune_empty_parents(from_path)
+          basename = to_key.split(".").last
+          Format.for(new_mentry.format).rewrite_name(to_path, basename)
+          etag_after = Value::Etag.for_file(to_path)
+
+          envelope = @reader.read(to_key)
+
+          extras = {
+            "from_key" => from_key, "to_key" => to_key,
+            "from_path" => from_path, "to_path" => to_path,
+            "uid" => envelope.uid
+          }
+          extras["correlation_id"] = @call.correlation_id if @call.correlation_id
+
+          @audit_log.append(
+            role: @call.role, verb: "key_mv", key: to_key,
+            etag_before: etag_before, etag_after: etag_after,
+            extras: extras
+          )
+
+          envelope
+        end
+
+        private
+
+        # After a file leaves a directory (delete or move-source), remove any
+        # now-empty parent dirs so bulk move/delete doesn't accrue orphan dirs
+        # (F3 of #161). Floored at the entry's *zone directory* — a zone is a
+        # declared, first-class container, so its own dir is preserved even when
+        # momentarily empty; only the sub-dirs the bulk op carved out are
+        # pruned. Stops at the first non-empty ancestor, so a dir holding a
+        # `.gitkeep` or sibling entries survives. Best-effort: a lost race or a
+        # non-empty dir is silently fine, never fatal to the write.
+        def prune_empty_parents(path)
+          floor = @geometry.lane_floor(path)
+          return unless floor
+
+          dir = File.dirname(path)
+          while dir.start_with?("#{floor}/") && Dir.empty?(dir)
+            Dir.rmdir(dir)
+            dir = File.dirname(dir)
+          end
+        rescue SystemCallError
+          nil
+        end
+
+        def enforce_name_match!(path, meta, format)
+          Textus::Format.for(format).enforce_name_match!(path, meta)
+        end
+
+        def serialize_for_put(mentry:, path:, meta:, body:, content:)
+          Textus::Format.for(mentry.format).serialize_for_put(
+            meta: meta, body: body, content: content, path: path,
+          )
+        end
+
+        def resolve_path(key)
+          @manifest.resolver.resolve(key).path
+        end
+
+        def serialize_entry(mentry, path, meta, payload, content)
+          serialize_for_put(
+            mentry: mentry, path: path,
+            meta: meta, body: payload.body, content: content
+          )
+        end
+
+        def validate_schema(mentry, eff_meta, eff_content)
+          schema = @schemas.fetch_or_nil(mentry.schema)
+          return unless schema
+
+          Format.for(mentry.format).validate_against(
+            schema,
+            { "_meta" => eff_meta, "content" => eff_content },
+          )
+        end
+
+        def check_etag!(path, key, if_etag)
+          etag_before = @file_store.exists?(path) ? @file_store.etag(path) : nil
+          raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
+
+          etag_before
+        end
+
+        def write_bytes(path, bytes)
+          @file_store.write(path, bytes)
+        end
+
+        def build_envelope(key, mentry, path, eff_meta, eff_body, eff_content, bytes = nil) # rubocop:disable Metrics/ParameterLists
+          raw = bytes || @file_store.read(path)
+          Textus::Value::Envelope.build(
+            key: key, mentry: mentry, path: path,
+            meta: eff_meta, body: eff_body,
+            etag: Value::Etag.for_bytes(raw),
+            content: eff_content
+          )
+        end
+
+        def audit_put(key, etag_before, etag_after)
+          extras = @call.correlation_id ? { "correlation_id" => @call.correlation_id } : nil
+          @audit_log.append(
+            role: @call.role, verb: "put", key: key,
+            etag_before: etag_before, etag_after: etag_after,
+            extras: extras
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/geometry.rb

@@ -0,0 +1,81 @@
+module Textus
+  class Store
+    class Geometry
+      RUN    = ".state"
+      DATA   = "data"
+      ASSETS = "assets"
+
+      def initialize(root)
+        @root = root
+        freeze
+      end
+
+      attr_reader :root
+
+      # -- data paths --
+      def data_root = File.join(@root, DATA)
+      def lane_path(lane_name) = File.join(data_root, lane_name.to_s)
+
+      def entry_path(mentry)
+        primary_ext = Format.for(mentry.format).extensions.first
+        rel = normalize_relative_path(mentry.path)
+        if File.extname(mentry.path) == ""
+          File.join(@root, rel + primary_ext)
+        else
+          File.join(@root, rel)
+        end
+      end
+
+      # -- runtime paths --
+      def run_root           = File.join(@root, RUN)
+      def cursor_path(role)  = File.join(run_root, "ephemeral", "cursors", role.to_s)
+      def lock_path(name)    = File.join(run_root, "ephemeral", "locks", "#{name}.lock")
+      def audit_dir_path     = File.join(run_root, "audit")
+      def audit_log_path     = File.join(audit_dir_path, "audit.log")
+      def sentinels_root = File.join(run_root, "tracking", "sentinels")
+      def store_db_path = File.join(run_root, "store.db")
+
+      # -- asset paths --
+      def asset_path(kind, date_str, zone, filename)
+        File.join(@root, ASSETS, kind, date_str, zone.to_s, filename)
+      end
+
+      # -- config paths --
+      def manifest_path = File.join(@root, "manifest.yaml")
+      def schemas_dir = File.join(@root, "schemas")
+      def schema_path(name) = File.join(schemas_dir, "#{name}.yaml")
+      def template_path(name) = File.join(@root, "templates", name)
+      def workflow_dir     = File.join(@root, "workflows")
+      def hooks_dir        = File.join(@root, "hooks")
+      def schemas_glob     = File.join(schemas_dir, "**", "*")
+
+      # -- gitignore --
+      def gitignore_body(untracked_entries: [])
+        lines = ["# textus runtime artifacts — safe to delete, never commit",
+                 "#{RUN}/"]
+        unless untracked_entries.empty?
+          lines << "# tracked:false entries — protocol-readable, not committed"
+          lines.concat(untracked_entries)
+        end
+        "#{lines.join("\n")}\n"
+      end
+
+      # -- lane boundary (replaces Writer#zone_floor) --
+      def lane_floor(path)
+        prefix = "#{data_root}/"
+        return nil unless path.start_with?(prefix)
+
+        seg = path.delete_prefix(prefix).split("/").first
+        seg && File.join(data_root, seg)
+      end
+
+      private
+
+      def normalize_relative_path(path)
+        return path if path.start_with?("data/")
+
+        File.join("data", path)
+      end
+    end
+  end
+end

data/lib/textus/store/index/builder.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Textus
+  class Store
+    module Index
+      class Builder
+        def initialize(store:)
+          @store = store
+        end
+
+        def rebuild!(resolver:)
+          rows = resolver.enumerate.filter_map { |row| build_row(row) }
+          now_iso = Time.now.utc.iso8601
+
+          @store.transaction do
+            @store.execute("DELETE FROM entries")
+            rows.each do |data|
+              @store.execute(
+                "INSERT INTO entries (key, lane, format, etag, content, extra, indexed_at)
+               VALUES (?, ?, ?, ?, ?, ?, ?)",
+                [data[:key], data[:lane], data[:format], data[:etag], data[:content], data[:extra], now_iso],
+              )
+            end
+            @store.execute("INSERT INTO entries_fts(entries_fts) VALUES('rebuild')")
+          end
+          { indexed: rows.size }
+        end
+
+        private
+
+        def build_row(row)
+          key = row.fetch(:key)
+          path = row.fetch(:path)
+          entry = row.fetch(:manifest_entry)
+          return nil unless path && File.file?(path)
+
+          raw = File.read(path)
+          parsed = Textus::Format.for(entry.format).parse(raw, path: path)
+          {
+            key: key,
+            lane: entry.lane,
+            format: entry.format.to_s,
+            etag: Textus::Value::Etag.for_bytes(raw),
+            content: content_text(parsed),
+            extra: extra_json(parsed),
+          }
+        end
+
+        def content_text(parsed)
+          content = parsed["content"]
+          body = parsed["body"]
+          parts = []
+          parts << body if body
+          parts << JSON.dump(content) if content
+          parts.join("\n")
+        end
+
+        def extra_json(parsed)
+          content = parsed["content"]
+          extra = {}
+          if content.is_a?(Hash)
+            extra["content_hash"] = content["content_hash"] if content["content_hash"]
+            url = content.dig("source", "url")
+            extra["url"] = url if url
+          end
+          JSON.dump(extra)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/index/lookup.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Textus
+  class Store
+    module Index
+      class Lookup
+        def initialize(store:)
+          @store = store
+        end
+
+        def search(query, lane: nil)
+          return [] if query.to_s.strip.empty?
+
+          clauses = ["entries_fts MATCH ?"]
+          params = [query]
+          if lane
+            clauses << "entries.lane = ?"
+            params << lane
+          end
+          conditions = "WHERE #{clauses.join(" AND ")}"
+          @store.execute(
+            "SELECT entries.key, entries.lane, entries.format, entries.etag, bm25(entries_fts) AS rank
+             FROM entries_fts JOIN entries ON entries_fts.rowid = entries.rowid
+           #{conditions}
+           ORDER BY rank",
+            params,
+          )
+        rescue SQLite3::SQLException
+          []
+        end
+
+        def find_by_hash(content_hash)
+          return nil if content_hash.to_s.empty?
+
+          find_extra("content_hash", content_hash)
+        end
+
+        def find_by_url(url)
+          return nil if url.to_s.empty?
+
+          find_extra("url", url)
+        end
+
+        private
+
+        def find_extra(name, value)
+          @store.execute("SELECT key, extra FROM entries ORDER BY indexed_at DESC").each do |row|
+            extra = JSON.parse(row["extra"] || "{}")
+            return row["key"] if extra[name] == value
+          end
+          nil
+        rescue SQLite3::SQLException
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/base.rb

@@ -0,0 +1,13 @@
+module Textus
+  class Store
+    module Jobs
+      class Base
+        def call(**)
+          raise NotImplementedError.new("#{self.class}#call")
+        end
+
+        def args = {}
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/index.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Index < Base
+        TYPE = "index"
+
+        def self.call(container:, call:) # rubocop:disable Lint/UnusedMethodArgument
+          Textus::Store::Index::Builder.new(store: container.job_store).rebuild!(resolver: container.manifest.resolver)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/materialize.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Materialize < Base
+        TYPE = "materialize"
+
+        def self.call(container:, call:, key:)
+          Textus::Produce::Engine.converge(container: container, call: call, keys: [key])
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/plan.rb

@@ -0,0 +1,11 @@
+module Textus
+  class Store
+    module Jobs
+      Plan = Data.define(:steps, :warnings) do
+        def to_h
+          { "steps" => steps, "warnings" => warnings }
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/jobs/planner.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Textus
+  class Store
+    module Jobs
+      class Planner
+        ACTIONS_BY_TRIGGER = {
+          "convergence" => %w[materialize sweep index],
+          "entry.written" => %w[materialize],
+          "entry.deleted" => %w[materialize],
+          "entry.moved" => %w[materialize],
+          "proposal.accepted" => %w[materialize],
+          "proposal.rejected" => %w[materialize],
+        }.freeze
+
+        SCOPE_RESOLVERS = {
+          "materialize" => :producible_keys,
+          "sweep" => :lane_keys,
+        }.freeze
+
+        GLOBAL_ACTIONS = {
+          "index" => {},
+          "sweep" => { "scope" => {} },
+        }.freeze
+
+        def self.seed(container:, queue:, role:)
+          jobs = new(container: container).plan(
+            trigger: { "type" => "convergence" },
+            role: role,
+          )
+          jobs.each { |j| queue.enqueue(j) }
+        end
+
+        def initialize(container:)
+          @container = container
+          @manifest  = container.manifest
+        end
+
+        def plan(trigger:, role:)
+          type = trigger["type"] || trigger[:type]
+          trigger["target"] || trigger[:target]
+          return [] if type.nil?
+
+          blocks_with_react = @manifest.rules.blocks.select(&:react)
+          if blocks_with_react.any?
+            plan_from_rules(blocks_with_react, type, role)
+          else
+            plan_from_defaults(type, role)
+          end
+        end
+
+        private
+
+        def plan_from_rules(blocks, type, role)
+          jobs = []
+          blocks
+            .select { |b| matches_trigger?(b.react, type) }
+            .each do |block|
+              do_action = block.react.raw["do"]
+              Array(do_action).each do |action|
+                if (global_args = GLOBAL_ACTIONS[action])
+                  jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: global_args, role: role)
+                else
+                  resolver = SCOPE_RESOLVERS.fetch(action, :producible_keys)
+                  keys = send(resolver, nil)
+                  keys.each { |key| jobs << job(action, key, role) }
+                end
+              end
+            end
+          jobs
+        end
+
+        def plan_from_defaults(type, role)
+          actions = ACTIONS_BY_TRIGGER.fetch(type, [])
+          jobs = []
+          producible_keys(nil).each { |k| jobs << job("materialize", k, role) } if actions.include?("materialize")
+          GLOBAL_ACTIONS.each do |action, args|
+            jobs << Textus::Store::Jobs::Queue::Job.new(type: action, args: args, role: role) if actions.include?(action)
+          end
+          jobs
+        end
+
+        def matches_trigger?(react, type)
+          on = react.raw["on"]
+          Array(on).include?(type)
+        end
+
+        def job(type, key, role)
+          Textus::Store::Jobs::Queue::Job.new(type: type, args: { "key" => key }, role: role)
+        end
+
+        def producible_keys(_target)
+          @manifest.data.entries
+                   .select { |e| !e.publish_tree.nil? || !e.publish_to.empty? }
+                   .map(&:key)
+        end
+
+        def lane_keys(_target)
+          @manifest.data.entries.map(&:key)
+        end
+      end
+    end
+  end
+end