textus 0.55.1 → 0.56.0

data/lib/textus/protocol/produce/render/context.rb

@@ -0,0 +1,43 @@
+module Textus
+  module Protocol
+    module Produce
+      class Render
+        class Context
+          def self.for(locals:, resolver: nil, from_path: nil, from_key: nil, edge_store: nil)
+            new(locals, resolver, from_path, from_key, edge_store)
+          end
+
+          def initialize(locals, resolver, from_path, from_key, edge_store)
+            @locals     = locals
+            @resolver   = resolver
+            @from_path  = from_path
+            @from_key   = from_key
+            @edge_store = edge_store
+          end
+
+          def to_erb_binding
+            mod = Module.new
+            @locals.each { |k, v| mod.define_method(k.to_sym) { v } }
+            obj = Object.new.extend(mod)
+
+            if @resolver && @from_path
+              resolver   = @resolver
+              from_path  = @from_path
+              from_key   = @from_key
+              edge_store = @edge_store
+              obj.define_singleton_method(:textus_link) do |key|
+                resolved = resolver.resolve(key: key, from_path: from_path)
+                edge_store&.record(from_key: from_key, to_key: key)
+                resolved
+              rescue Textus::Protocol::Links::Resolver::UnknownKeyError
+                "`textus get #{key}`"
+              end
+            end
+
+            obj.instance_eval { binding }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/produce/render.rb

@@ -0,0 +1,44 @@
+module Textus
+  module Protocol
+    module Produce
+      class Render
+        def initialize(template_loader:, manifest: nil, source_publish_path: nil, entry_key: nil, edge_store: nil)
+          @template_loader     = template_loader
+          @manifest            = manifest
+          @source_publish_path = source_publish_path
+          @entry_key           = entry_key
+          @edge_store          = edge_store
+        end
+
+        def bytes_for(target:, data:, boot:)
+          raise ArgumentError.new("Produce::Render called for a verbatim target #{target.to.inspect}") unless target.renders?
+
+          locals   = target.inject_boot ? data.merge("boot" => boot) : data
+          resolver = @manifest ? Textus::Protocol::Links::Resolver.new(manifest: @manifest) : nil
+          ctx      = Render::Context.for(
+            locals: locals,
+            resolver: resolver,
+            from_path: target.to,
+            from_key: @entry_key,
+            edge_store: @edge_store,
+          )
+          raw = ERB.new(@template_loader.call(target.template), trim_mode: "-").result(ctx.to_erb_binding)
+          rewrite(raw, target.to, resolver)
+        end
+
+        private
+
+        def rewrite(bytes, from_path, resolver)
+          return bytes unless resolver && bytes.include?("textus:")
+
+          Textus::Protocol::Links::UriRewriter.new(
+            resolver: resolver,
+            from_path: from_path,
+            from_key: @entry_key,
+            edge_store: @edge_store,
+          ).rewrite(bytes)
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/schema/registry.rb

@@ -0,0 +1,44 @@
+module Textus
+  module Protocol
+    class Schema
+      class Registry
+        def initialize(dir)
+          @dir = dir
+          @schemas = {}
+          load_all
+        end
+
+        def fetch(name)
+          @schemas[name] || raise(Textus::IoError.new("schema not found: #{File.join(@dir, "#{name}.yaml")}"))
+        end
+
+        def fetch_or_nil(name)
+          return nil if name.nil?
+
+          fetch(name)
+        end
+
+        def all
+          @schemas.values
+        end
+
+        def by_name
+          @schemas.dup
+        end
+
+        private
+
+        def load_all
+          return unless File.directory?(@dir)
+
+          Dir.glob(File.join(@dir, "**", "*.yaml")).each do |path|
+            name = path.delete_prefix("#{@dir}/").sub(/\.yaml\z/, "")
+            @schemas[name] = Textus::Protocol::Schema.load(path)
+          rescue StandardError => e
+            warn "textus: failed to load schema '#{name}' at #{path}: #{e.message}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/schema/tools.rb

@@ -0,0 +1,104 @@
+require "yaml"
+require "fileutils"
+
+module Textus
+  module Protocol
+    class Schema
+      module Tools
+        def self.init(store, name:, from:)
+          env = pure_get(store, Textus::Value::Role::DEFAULT, from)
+          meta = env["_meta"]
+          schema = {
+            "name" => name,
+            "required" => meta.keys,
+            "optional" => [],
+            "fields" => meta.each_with_object({}) { |(k, v), h| h[k] = { "type" => infer_type(v) } },
+          }
+          geom = Textus::Protocol::Layout.new(store.root)
+          FileUtils.mkdir_p(geom.schemas_dir)
+          target = geom.schema_path(name)
+          File.write(target, YAML.dump(schema))
+          { "protocol" => PROTOCOL, "schema_name" => name, "path" => target }
+        end
+
+        def self.diff(store, name:)
+          schema = load_schema(store, name)
+          drift = []
+          store.manifest.resolver.enumerate.each do |row|
+            env = pure_get(store, Textus::Value::Role::DEFAULT, row[:key])
+            begin
+              schema.validate!(env["_meta"])
+            rescue SchemaViolation => e
+              drift << { "key" => row[:key], "details" => e.details }
+            end
+          end
+          { "protocol" => PROTOCOL, "schema_name" => name, "drift" => drift }
+        end
+
+        def self.migrate(store, name:, rename: nil)
+          renames =
+            if rename
+              old_field, new_field = rename.split(":", 2)
+              raise UsageError.new("--rename=OLD:NEW") unless old_field && new_field && !new_field.empty?
+
+              { old_field => new_field }
+            else
+              load_schema(store, name).evolution["migrate_from"] || {}
+            end
+          raise UsageError.new("schema migrate needs --rename=OLD:NEW or schema.evolution.migrate_from") if renames.empty?
+
+          authority = accept_role_for(store)
+          ops = store.with_role(authority)
+          touched = []
+          store.manifest.resolver.enumerate.each do |row|
+            env = pure_get(store, authority, row[:key])
+            meta = (env["_meta"] || {}).dup
+            changed = false
+            renames.each do |old, new|
+              if meta.key?(old)
+                meta[new] = meta.delete(old)
+                changed = true
+              end
+            end
+            next unless changed
+
+            ops.put(key: row[:key], meta: meta, body: env["body"])
+            touched << row[:key]
+          end
+          { "protocol" => PROTOCOL, "migrated" => touched, "renames" => renames }
+        end
+
+        def self.infer_type(value)
+          case value
+          when String  then "string"
+          when Numeric then "number"
+          when true, false then "boolean"
+          when Array   then "array"
+          when Hash    then "object"
+          else "string"
+          end
+        end
+
+        def self.pure_get(store, role, key)
+          store.with_role(role).get(key: key)
+        end
+
+        def self.load_schema(store, name)
+          store.schemas.fetch(name)
+        rescue IoError
+          raise UsageError.new("schema not found: #{name}")
+        end
+
+        def self.accept_role_for(store)
+          authority = store.manifest.policy.writers_for("knowledge").first
+          return authority if authority
+
+          raise UsageError.new(
+            "schema migrate requires a role that can write knowledge; " \
+            "none found in the fixed lane table",
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/schema.rb

@@ -0,0 +1,117 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    class Schema
+      attr_reader :name, :required, :optional, :fields, :raw
+
+      def self.load(path)
+        raw = YAML.safe_load_file(path, permitted_classes: [Symbol, Date, Time], aliases: false)
+        new(raw)
+      end
+
+      def initialize(raw)
+        @raw = raw || {}
+        @name = @raw["name"]
+        @required = Array(@raw["required"])
+        @optional = Array(@raw["optional"])
+        @fields = @raw["fields"] || {}
+      end
+
+      def to_h
+        @raw
+      end
+
+      def maintained_by(field)
+        meta = @fields[field] or return nil
+        meta["maintained_by"]
+      end
+
+      def unowned_fields
+        @fields.each_with_object([]) do |(name, spec), acc|
+          next unless spec.is_a?(Hash)
+
+          acc << name if spec["maintained_by"].nil?
+        end
+      end
+
+      def evolution
+        raw = @raw["evolution"] || {}
+        raw.each_with_object({}) do |(k, v), h|
+          h[k] = v.is_a?(Date) || v.is_a?(Time) ? v.to_s : v
+        end
+      end
+
+      def validate!(frontmatter)
+        missing = @required - frontmatter.keys
+        raise Textus::SchemaViolation.new("missing" => missing) unless missing.empty?
+
+        known = (@required + @optional).uniq
+        frontmatter.each do |k, v|
+          next unless @fields.key?(k)
+
+          check_type!(k, v, @fields[k])
+        end
+
+        warnings = frontmatter.keys - known
+        warnings.map { |w| "unknown field: #{w}" }
+      end
+
+      TYPE_VALIDATORS = {
+        "string" => :validate_string,
+        "number" => :validate_number,
+        "boolean" => :validate_boolean,
+        "enum" => :validate_enum,
+        "array" => :validate_array,
+        "object" => :validate_object,
+      }.freeze
+
+      private
+
+      def check_type!(field, value, spec)
+        type = spec["type"]
+        return unless type
+
+        send(TYPE_VALIDATORS[type], field, value, spec)
+      end
+
+      def validate_string(field, value, spec)
+        bad!(field, "expected string") unless value.is_a?(String)
+        return unless (max = spec["max"]) && value.bytesize > max
+
+        bad!(field, "exceeds max #{max}")
+      end
+
+      def validate_number(field, value, _spec)
+        bad!(field, "expected number") unless value.is_a?(Numeric)
+      end
+
+      def validate_boolean(field, value, _spec)
+        bad!(field, "expected boolean") unless [true, false].include?(value)
+      end
+
+      def validate_enum(field, value, spec)
+        values = Array(spec["values"])
+        bad!(field, "not in enum #{values.inspect}") unless values.include?(value)
+      end
+
+      def validate_array(field, value, spec)
+        bad!(field, "expected array") unless value.is_a?(Array)
+        return unless (items = spec["items"])
+
+        value.each_with_index { |v, i| check_type!("#{field}[#{i}]", v, items) }
+      end
+
+      def validate_object(field, value, spec)
+        bad!(field, "expected object") unless value.is_a?(Hash)
+        return unless (sub = spec["fields"])
+
+        sub.each { |fk, fspec| check_type!("#{field}.#{fk}", value[fk], fspec) if value.key?(fk) }
+      end
+
+      def bad!(field, reason)
+        raise Textus::SchemaViolation.new("field" => field, "reason" => reason)
+      end
+    end
+  end
+end

data/lib/textus/protocol/session/bridge.rb

@@ -0,0 +1,11 @@
+Textus::Protocol::Session::Bridge = Data.define(:container, :gate) do
+  def store         = container.store
+  def manifest      = container.manifest
+  def schemas       = container.schemas
+  def layout        = container.layout
+  def workflows     = container.workflows
+  def root          = container.root
+  def ttl_evaluator = container.ttl_evaluator
+
+  def dispatch(...) = gate.dispatch(...)
+end

data/lib/textus/protocol/session/context.rb

@@ -0,0 +1,5 @@
+Textus::Protocol::Session::Context = Data.define(:role, :correlation_id, :dry_run, :cursor, :contract_etag, :propose_lane) do
+  def call
+    Textus::Value::Call.build(role:, correlation_id:)
+  end
+end

data/lib/textus/protocol/session.rb

@@ -0,0 +1,87 @@
+require "forwardable"
+
+module Textus
+  module Protocol
+    class Session
+      extend Forwardable
+
+      attr_reader :context, :bridge
+
+      DOMAIN_VERBS = (VerbRegistry::ENTRY_VERBS + VerbRegistry::OPS_VERBS).to_set.freeze
+
+      def_delegators :context, :role, :cursor, :correlation_id, :contract_etag, :propose_lane
+      def_delegator  :context, :dry_run, :dry_run?
+      def_delegators :bridge, :root, :container, :store, :manifest, :schemas, :layout, :workflows, :gate, :ttl_evaluator
+
+      def self.open(root, role: Value::Role::DEFAULT, correlation_id: nil, dry_run: false)
+        root = File.expand_path(root)
+        container = Protocol::Builder.new.call(root)
+        gate = container.gate
+        bridge = Session::Bridge.new(container:, gate:)
+        context = Session::Context.new(
+          role: role.to_s,
+          correlation_id: correlation_id || SecureRandom.uuid,
+          dry_run: dry_run,
+          cursor: container.store.audit_log.latest[:seq],
+          contract_etag: Value::Etag.for_contract(bridge.layout),
+          propose_lane: container.manifest.policy.propose_lane_for(role.to_s),
+        )
+        new(bridge:, context:)
+      end
+
+      def initialize(bridge:, context:)
+        @bridge = bridge
+        @context = context
+      end
+
+      def method_missing(name, *args, **kwargs)
+        if DOMAIN_VERBS.include?(name)
+          raise ArgumentError.new("#{name} accepts keyword arguments only") unless args.empty?
+
+          return bridge.dispatch(role.to_sym, name, kwargs, call: context.call)
+        end
+
+        return bridge.public_send(name, *args, **kwargs) if bridge.respond_to?(name)
+
+        super
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        DOMAIN_VERBS.include?(name) || bridge.respond_to?(name) || super
+      end
+
+      def with_correlation_id(cid)
+        self.class.new(bridge:, context: context.with(correlation_id: cid))
+      end
+
+      def with_role(new_role)
+        self.class.new(
+          bridge:,
+          context: context.with(
+            role: new_role.to_s,
+            cursor: container.store.audit_log.latest[:seq],
+            contract_etag: Value::Etag.for_contract(container.layout),
+            propose_lane: manifest.policy.propose_lane_for(new_role.to_s),
+          ),
+        )
+      end
+
+      def advance_cursor(new_cursor)
+        @context = context.with(cursor: new_cursor)
+        self
+      end
+
+      def check_etag!(observed_etag)
+        return if observed_etag == contract_etag
+
+        raise Textus::ContractDrift.new(
+          "contract changed (was #{short_etag(contract_etag)}, now #{short_etag(observed_etag)})",
+        )
+      end
+
+      private
+
+      def short_etag(etag) = etag.to_s.delete_prefix("sha256:")[0, 8]
+    end
+  end
+end

data/lib/textus/protocol/store_engine/delete.rb

@@ -0,0 +1,37 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Delete
+        module_function
+
+        def call(key, deps:, if_etag: nil)
+          path = Pipeline.resolve_path(key, deps)
+          assert_exists!(key, path, deps)
+          etag_before = Pipeline.check_etag(key, path, if_etag, deps.file_system)
+          deps.file_system.delete(path)
+          deps.entry_index.remove(key:)
+          Pipeline.prune_parents(path, deps)
+          append_audit(key, etag_before, deps)
+          Pipeline.emit_event(deps, event_type: "entry.deleted",
+                                    pending_key: key, emit_key: key)
+          nil
+        end
+
+        def assert_exists!(key, path, deps)
+          return if deps.file_system.exists?(path)
+
+          raise Textus::UnknownKey.new(key,
+                                       suggestions: deps.manifest.resolver.suggestions_for(key))
+        end
+
+        def append_audit(key, etag_before, deps)
+          deps.audit_log.append(
+            role: deps.call.role, verb: "key_delete", key:,
+            etag_before:, etag_after: nil,
+            correlation_id: deps.call.correlation_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine/move.rb

@@ -0,0 +1,109 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Move
+        MoveResult = Data.define(:from_key, :to_key, :from_path, :to_path, :dry_run, :envelope)
+
+        module_function
+
+        def call(from_key:, to_key:, deps:, if_etag: nil, dry_run: false)
+          validate!(from_key, to_key)
+          from_path, to_path, from_entry, to_entry = resolve_paths(from_key, to_key, deps)
+          assert_source_exists!(from_key, from_path, deps)
+          assert_target_not_exists!(to_path)
+          check_cross_zone!(from_entry, to_entry)
+          ensure_uid!(from_key, from_entry, from_path, deps)
+          etag_before = Pipeline.check_etag(from_key, from_path, if_etag, deps.file_system)
+
+          if dry_run
+            return MoveResult.new(from_key:, to_key:, from_path:, to_path:,
+                                  dry_run: true, envelope: nil)
+          end
+
+          deps.file_system.mv(from_path, to_path)
+          Pipeline.prune_parents(from_path, deps)
+          rewrite_basename!(to_key, to_path, to_entry)
+          etag_after = Value::Etag.for_file(to_path)
+          envelope = deps.reader.read(key: to_key)
+          append_audit(to_key, etag_before, etag_after, deps)
+          reindex_entry(from_key, to_key, to_entry, envelope, deps)
+          Pipeline.emit_event(deps, event_type: "entry.moved",
+                                    pending_key: from_key, emit_key: to_key,
+                                    emit_lane: to_entry.lane,
+                                    pending_keys: [from_key, to_key])
+          MoveResult.new(from_key:, to_key:, from_path:, to_path:,
+                         dry_run: false, envelope:)
+        end
+
+        def validate!(from_key, to_key)
+          Manifest::Data.validate_key!(from_key)
+          Manifest::Data.validate_key!(to_key)
+          raise UsageError.new("old and new keys are identical") if from_key == to_key
+        end
+
+        def resolve_paths(from_key, to_key, deps)
+          from = deps.manifest.resolver.resolve(from_key)
+          to   = deps.manifest.resolver.resolve(to_key)
+          [from.path, to.path, from.entry, to.entry]
+        end
+
+        def assert_source_exists!(key, path, deps)
+          return if deps.file_system.exists?(path)
+
+          raise UnknownKey.new(key, suggestions: deps.manifest.resolver.suggestions_for(key))
+        end
+
+        def assert_target_not_exists!(path)
+          return unless path && File.exist?(path)
+
+          raise UsageError.new("target already exists")
+        end
+
+        def check_cross_zone!(from_entry, to_entry)
+          return if from_entry.lane == to_entry.lane
+
+          raise UsageError.new("cross-zone moves forbidden (#{from_entry.lane} → #{to_entry.lane})")
+        end
+
+        def ensure_uid!(_key, entry, path, deps)
+          return if Infra::Store::Envelope::Meta::NO_META_FORMATS.include?(entry.format)
+          return unless path && deps.file_system.exists?(path)
+
+          raw = deps.file_system.read(path)
+          parsed = Format.for(entry.format).parse(raw, path:)
+          meta = parsed["_meta"] || {}
+          return if meta["uid"].is_a?(String) && !meta["uid"].empty?
+
+          new_meta, = Infra::Store::Envelope::Meta.inject_all(meta, parsed["content"], {},
+                                                              format: entry.format)
+          bytes, = Format.for(entry.format).serialize_for_put(
+            meta: new_meta, body: parsed["body"] || "", content: parsed["content"], path:,
+          )
+          deps.file_system.write(path, bytes)
+        end
+
+        def rewrite_basename!(key, path, entry)
+          basename = key.split(".").last
+          Format.for(entry.format).rewrite_name(path, basename)
+        end
+
+        def append_audit(key, etag_before, etag_after, deps)
+          deps.audit_log.append(
+            role: deps.call.role, verb: "key_mv", key:,
+            etag_before:, etag_after:,
+            correlation_id: deps.call.correlation_id
+          )
+        end
+
+        def reindex_entry(from_key, to_key, to_entry, envelope, deps)
+          deps.entry_index.remove(key: from_key)
+          content = envelope.content ? envelope.content.to_json : envelope.body
+          deps.entry_index.record(
+            key: to_key, lane: to_entry.lane, format: to_entry.format,
+            schema: to_entry.schema, content:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine/pipeline.rb

@@ -0,0 +1,61 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Pipeline
+        WriteDeps = Data.define(:file_system, :manifest, :store, :call, :reader, :layout, :event_emitter, :schemas) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def file_store   = store.file_store
+        end
+
+        DeleteDeps = Data.define(:file_system, :manifest, :store, :call, :layout, :event_emitter) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def workflow_queue = store.workflow_queue
+        end
+
+        MoveDeps = Data.define(:file_system, :manifest, :store, :call, :reader, :layout, :event_emitter) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def workflow_queue = store.workflow_queue
+        end
+
+        module_function
+
+        def resolve_path(key, deps)
+          deps.manifest.resolver.resolve(key).path
+        end
+
+        def check_etag(key, path, if_etag, file_system)
+          etag_before = file_system.exists?(path) ? file_system.etag(path) : nil
+          raise Textus::EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
+
+          etag_before
+        end
+
+        def prune_parents(path, deps)
+          floor = deps.layout.lane_floor(path)
+          return unless floor
+
+          dir = File.dirname(path)
+          while dir.start_with?("#{floor}/") && deps.file_system.dir_empty?(dir)
+            deps.file_system.rmdir(dir)
+            dir = File.dirname(dir)
+          end
+        rescue SystemCallError
+          # race: dir may have been emptied by another writer
+        end
+
+        def emit_event(deps, event_type:, pending_key:, emit_key:, emit_lane: nil, pending_keys: [])
+          return unless deps.event_emitter
+          return if deps.store.workflow_queue.pending?(event_type:, key: pending_key)
+          return if pending_keys.any? { |k| deps.store.workflow_queue.pending?(event_type:, key: k) }
+
+          deps.event_emitter.emit(event_type:, key: emit_key, lane: emit_lane || "artifacts")
+        rescue StandardError => e
+          warn "textus: emit_workflow_event(#{event_type}, #{emit_key}) failed: #{e.message}"
+        end
+      end
+    end
+  end
+end