textus 0.55.1 → 0.56.0

data/lib/textus/protocol/format/base.rb

@@ -0,0 +1,80 @@
+module Textus
+  module Protocol
+    module Format
+      class Base
+        def self.parse(_raw, path: nil)
+          _ = path
+          raise NotImplementedError.new("#{name}.parse not implemented")
+        end
+
+        def self.serialize(meta: {}, body: "", content: nil)
+          _ = meta
+          _ = body
+          _ = content
+          raise NotImplementedError.new("#{name}.serialize not implemented")
+        end
+
+        def self.extensions
+          raise NotImplementedError.new("#{name}.extensions not implemented")
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["_meta"] || {})
+        end
+
+        def self.nested_glob
+          raise NotImplementedError.new("#{name}.nested_glob not implemented")
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("#{format_name} nested path must not have an extension")
+          end
+
+          return if extensions.include?(ext)
+
+          raise UsageError.new("#{format_name} format requires '#{extensions.join("' or '")}' path (got #{ext.inspect})")
+        end
+
+        def self.enforce_name_match!(path, meta)
+          return unless meta.is_a?(Hash) && meta["name"]
+
+          ext = extensions.first
+          basename = File.basename(path, ext)
+          return if meta["name"] == basename
+
+          raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
+        end
+
+        def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
+          raw = File.binread(path)
+          parsed = parse(raw, path: path)
+          meta = parsed["_meta"] || {}
+          return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
+
+          new_meta = meta.merge("name" => basename)
+          File.binwrite(path, serialize(meta: new_meta, body: parsed["body"] || "", content: parsed["content"]))
+          true
+        end
+
+        def self.format_name
+          name.split("::").last.downcase
+        end
+
+        def self.validate_raw_entry!(_parsed, _lane); end
+
+        def self.nested_ext(_segment = nil) = extensions.first
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          _ = path
+          _ = content
+          bytes = serialize(meta: meta || {}, body: body.to_s)
+          [bytes, meta, body.to_s, nil]
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/bash.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Bash < Script
+        def self.extensions = [".sh"]
+        def self.nested_ext(*) = ".sh"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/javascript.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Javascript < Script
+        def self.extensions = [".js"]
+        def self.nested_ext(*) = ".js"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/json.rb

@@ -0,0 +1,82 @@
+require "json"
+
+module Textus
+  module Protocol
+    module Format
+      class Json < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          begin
+            parsed = ::JSON.parse(raw)
+          rescue ::JSON::ParserError => e
+            raise BadFrontmatter.new(path, "JSON parse failed: #{e.message}")
+          end
+          raise BadFrontmatter.new(path, "JSON top-level must be an object") unless parsed.is_a?(Hash)
+
+          meta = parsed["_meta"]
+          fm = meta.is_a?(Hash) ? meta : {}
+          content_without_meta = parsed.except("_meta")
+          { "_meta" => fm, "body" => raw, "content" => content_without_meta }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          if content.is_a?(Hash)
+            on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+            out = ::JSON.pretty_generate(on_disk)
+            out += "\n" unless out.end_with?("\n")
+            out
+          elsif body && !body.to_s.empty?
+            b = body.to_s
+            b += "\n" unless b.end_with?("\n")
+            b
+          else
+            raise UsageError.new("json serialize requires :content or :body")
+          end
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["content"] || {})
+        end
+
+        def self.extensions = [".json"]
+        def self.nested_glob = "**/*.json"
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          raise UsageError.new("put for json requires content: or body:") if content.nil? && (body.nil? || body.to_s.empty?)
+
+          if content.nil?
+            begin
+              parsed = parse(body.to_s, path: path)
+            rescue BadFrontmatter => e
+              raise BadContent.new(path, "bad_content: #{e.message}")
+            end
+            [body.to_s, parsed["_meta"], body.to_s, parsed["content"]]
+          else
+            bytes = serialize(meta: meta, body: "", content: content)
+            [bytes, meta, bytes, content]
+          end
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested json path must not have an extension")
+          end
+
+          return if ext == ".json"
+
+          raise UsageError.new("json format requires '.json' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/markdown.rb

@@ -0,0 +1,53 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    module Format
+      class Markdown < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+          return { "_meta" => {}, "body" => raw, "content" => nil } unless raw.start_with?("---\n") || raw.start_with?("---\r\n")
+
+          lines = raw.split(/\r?\n/, -1)
+          close_idx = lines[1..].index("---")
+          raise BadFrontmatter.new(path, "frontmatter not terminated") unless close_idx
+
+          close_idx += 1
+          fm_yaml = lines[1...close_idx].join("\n")
+          body = lines[(close_idx + 1)..].join("\n")
+          begin
+            fm = fm_yaml.strip.empty? ? {} : ::YAML.safe_load(fm_yaml, permitted_classes: [Date, Time], aliases: false)
+          rescue Psych::SyntaxError => e
+            raise BadFrontmatter.new(path, "YAML parse failed: #{e.message}")
+          end
+          fm = {} unless fm.is_a?(Hash)
+          { "_meta" => fm, "body" => body, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = content
+          fm_yaml = meta.empty? ? "" : ::YAML.dump(meta).sub(/\A---\n/, "")
+          body = body.to_s
+          body += "\n" unless body.empty? || body.end_with?("\n")
+          "---\n#{fm_yaml}---\n#{body}"
+        end
+
+        def self.extensions = [".md"]
+        def self.nested_glob = "**/*.md"
+
+        def self.validate_path_extension(path, _nested)
+          ext = File.extname(path)
+          return if ["", ".md"].include?(ext)
+
+          raise UsageError.new("markdown format requires '.md' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/python.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Python < Script
+        def self.extensions = [".py"]
+        def self.nested_ext(*) = ".py"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/ruby.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Ruby < Script
+        def self.extensions = [".rb"]
+        def self.nested_ext(*) = ".rb"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/script.rb

@@ -0,0 +1,55 @@
+module Textus
+  module Protocol
+    module Format
+      class Script < Base
+        EXTENSIONS = %w[.rb .py .sh .js .pl .rs .exs].freeze
+
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          { "_meta" => {}, "body" => raw, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = meta
+          _ = content
+          b = body.to_s
+          b += "\n" unless b.empty? || b.end_with?("\n")
+          b
+        end
+
+        def self.extensions = EXTENSIONS
+        def self.nested_glob = "**/*"
+        def self.nested_ext(*) = ""
+        def self.enforce_name_match!(_path, _meta); end
+
+        def self.rewrite_name(_path, _basename)
+          false
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested script path must not have an extension")
+          end
+
+          return if EXTENSIONS.include?(ext) || ext == ""
+
+          raise UsageError.new("script format requires #{EXTENSIONS.join(" | ")} extension (got #{ext.inspect})")
+        end
+
+        def self.validate_raw_entry!(_parsed, lane)
+          nil unless lane == "scratchpad"
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/text.rb

@@ -0,0 +1,48 @@
+module Textus
+  module Protocol
+    module Format
+      class Text < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          { "_meta" => {}, "body" => raw, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = meta
+          _ = content
+          b = body.to_s
+          b += "\n" unless b.empty? || b.end_with?("\n")
+          b
+        end
+
+        def self.extensions = [".txt"]
+        def self.nested_glob = "**/*.txt"
+        def self.enforce_name_match!(_path, _meta); end
+
+        def self.rewrite_name(_path, _basename) # rubocop:disable Naming/PredicateMethod
+          false
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested text path must not have an extension")
+          end
+
+          return if [".txt", ""].include?(ext)
+
+          raise UsageError.new("text format requires '.txt' or no extension (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/yaml.rb

@@ -0,0 +1,105 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    module Format
+      class Yaml < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          begin
+            parsed = ::YAML.safe_load(raw, permitted_classes: [Date, Time], aliases: false)
+          rescue Psych::SyntaxError, Psych::AliasesNotEnabled, Psych::DisallowedClass => e
+            raise BadFrontmatter.new(path, "YAML parse failed: #{e.message}")
+          end
+          raise BadFrontmatter.new(path, "YAML top-level must be a mapping") unless parsed.is_a?(Hash)
+
+          meta = parsed["_meta"]
+          fm = meta.is_a?(Hash) ? meta : {}
+          content_without_meta = parsed.except("_meta")
+          { "_meta" => fm, "body" => raw, "content" => content_without_meta }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          if content.is_a?(Hash)
+            on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+            ::YAML.dump(on_disk).sub(/\A---\n/, "")
+          elsif body && !body.to_s.empty?
+            b = body.to_s
+            b += "\n" unless b.end_with?("\n")
+            b
+          else
+            raise UsageError.new("yaml serialize requires :content or :body")
+          end
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["content"] || {})
+        end
+
+        RAW_REQUIRED = %w[ingested_at content_hash].freeze
+        RAW_SOURCE_KINDS = %w[url file asset].freeze
+
+        def self.validate_raw_entry!(parsed, lane)
+          return unless lane == "raw"
+
+          content = parsed["content"] || {}
+          return if content["superseded_by"]
+
+          missing = RAW_REQUIRED.reject { |f| content[f] }
+          raise Textus::BadContent.new(nil, "raw entry missing required field(s): #{missing.join(", ")}") if missing.any?
+
+          source = content["source"] || {}
+          kind = source["kind"]
+          unless RAW_SOURCE_KINDS.include?(kind)
+            raise Textus::BadContent.new(
+              nil, "raw entry source.kind must be #{RAW_SOURCE_KINDS.join("|")}, got #{kind.inspect}"
+            )
+          end
+
+          return unless kind == "url" && !source["url"]
+
+          raise Textus::BadContent.new(nil, "raw entry with source.kind=url must have source.url")
+        end
+
+        def self.extensions = [".yaml", ".yml"]
+        def self.nested_glob = "**/*.{yaml,yml}"
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          raise UsageError.new("put for yaml requires content: or body:") if content.nil? && (body.nil? || body.to_s.empty?)
+
+          if content.nil?
+            begin
+              parsed = parse(body.to_s, path: path)
+            rescue BadFrontmatter => e
+              raise BadContent.new(path, "bad_content: #{e.message}")
+            end
+            [body.to_s, parsed["_meta"], body.to_s, parsed["content"]]
+          else
+            bytes = serialize(meta: meta, body: "", content: content)
+            [bytes, meta, bytes, content]
+          end
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested yaml path must not have an extension")
+          end
+
+          return if [".yaml", ".yml"].include?(ext)
+
+          raise UsageError.new("yaml format requires '.yaml' or '.yml' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format.rb

@@ -0,0 +1,73 @@
+module Textus
+  module Protocol
+    module Format
+      SEP = "---".freeze
+
+      STRATEGIES = {
+        "markdown" => -> { Protocol::Format::Markdown },
+        "json" => -> { Protocol::Format::Json },
+        "yaml" => -> { Protocol::Format::Yaml },
+        "text" => -> { Protocol::Format::Text },
+        "script" => -> { Protocol::Format::Script },
+        "bash" => -> { Protocol::Format::Bash },
+        "ruby" => -> { Protocol::Format::Ruby },
+        "python" => -> { Protocol::Format::Python },
+        "javascript" => -> { Protocol::Format::Javascript },
+      }.freeze
+
+      @registry = nil
+
+      def self.registry=(reg)
+        @registry = reg
+      end
+
+      def self.registry
+        @registry
+      end
+
+      EXT_TO_FORMAT = {
+        ".md" => "markdown",
+        ".json" => "json",
+        ".yaml" => "yaml",
+        ".yml" => "yaml",
+        ".txt" => "text",
+        ".rb" => "ruby",
+        ".py" => "python",
+        ".sh" => "bash",
+        ".js" => "javascript",
+        ".pl" => "script",
+        ".rs" => "script",
+        ".exs" => "script",
+      }.freeze
+
+      def self.for(format)
+        key = format.to_s
+        return registry.fetch(key).call if registry&.key?(key)
+
+        STRATEGIES.fetch(key) { raise Textus::UsageError.new("unknown entry format: #{format.inspect}") }.call
+      end
+
+      def self.infer_from_extension(ext)
+        EXT_TO_FORMAT[ext]
+      end
+
+      def self.formats
+        EXT_TO_FORMAT.values.uniq
+      end
+
+      def self.parse(raw, path: nil, format: "markdown")
+        Protocol::Format.for(format).parse(raw, path: path)
+      end
+
+      def self.serialize(meta: {}, body: "", content: nil, format: "markdown")
+        Protocol::Format.for(format).serialize(meta: meta, body: body, content: content)
+      end
+
+      def self.data_to_payload(data, format)
+        return { meta: {}, body: "", content: nil } if data.nil?
+
+        Protocol::Format.for(format).data_to_payload(data)
+      end
+    end
+  end
+end

data/lib/textus/protocol/freshness/ttl_evaluator.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Textus
+  module Protocol
+    module Freshness
+      class TtlEvaluator
+        def initialize(manifest:, file_stat:, clock:)
+          @manifest  = manifest
+          @file_stat = file_stat
+          @clock     = clock
+        end
+
+        def verdict(mentry)
+          ttl = lane_ttl(mentry.lane)
+          return fresh if ttl.nil?
+
+          stale = age_stale?(file_basis(mentry), ttl)
+          Verdict.build(stale: stale, reason: stale ? "ttl exceeded" : nil, fetching: false)
+        end
+
+        def stale_keys(prefix: nil, lane: nil)
+          @manifest.data.entries.select { |m| due?(m, prefix: prefix, lane: lane) }.map(&:key)
+        end
+
+        private
+
+        def fresh = Verdict.build(stale: false, reason: nil, fetching: false)
+
+        def file_basis(mentry)
+          path = @manifest.resolver.resolve(mentry.key).path
+          return nil unless @file_stat.exists?(path)
+
+          @file_stat.mtime(path)
+        end
+
+        def lane_ttl(lane)
+          Textus::Protocol::V4::LANES.dig(lane, :retention_ttl)
+        end
+
+        def due?(mentry, prefix:, lane:)
+          return false if lane && mentry.lane != lane
+          return false if prefix && !mentry.key.start_with?(prefix)
+
+          ttl = lane_ttl(mentry.lane)
+          return false if ttl.nil?
+
+          path = @manifest.resolver.resolve(mentry.key).path
+          return true unless @file_stat.exists?(path)
+
+          age_stale?(file_basis(mentry), ttl)
+        end
+
+        def age_stale?(basis, ttl)
+          return true if basis.nil?
+
+          (@clock.now - basis).to_i > ttl
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/gate.rb

@@ -0,0 +1,61 @@
+module Textus
+  module Protocol
+    class Gate
+      HANDLER_GROUPS = {
+        read: Handlers::Read,
+        write: Handlers::Write,
+        maintenance: Handlers::System,
+      }.freeze
+
+      def initialize(manifest:, resolver:, container:, store_engine:, policy: nil, async: nil)
+        @manifest = manifest
+        @resolver = resolver
+        @container = container
+        @store_engine = store_engine
+        @async = async
+        @policy = policy || manifest.policy
+      end
+
+      def dispatch(role, verb, params = {}, call: nil)
+        spec = VerbRegistry.for(verb)
+        return error(:unknown_verb, "Unknown verb: #{verb}") unless spec
+
+        key = params[:key]
+        authorize!(role, verb, key, spec)
+
+        call_obj = call || Value::Call.build(role:, correlation_id: SecureRandom.uuid)
+        bound = Protocol::Binder.bind(spec, params)
+        ctx_class = spec.read? && !spec.system? ? Handlers::QueryContext : Handlers::CommandContext
+        ctx = ctx_class.new(@store_engine, @container, call_obj)
+
+        handler = spec.lane ? Lane.handler_for(spec.lane) : handler_group(spec)
+        handler.public_send(verb, **bound, ctx:, call: call_obj)
+      end
+
+      private
+
+      def handler_group(spec)
+        return Handlers::System if spec.system?
+
+        HANDLER_GROUPS[spec.category] || Handlers::System
+      end
+
+      def authorize!(role, _verb, key, spec)
+        return true unless spec&.write?
+
+        key_lane = key&.split(".")&.first
+        lane = key_lane || spec.lane&.to_s
+        return true unless lane
+
+        lane_map = Textus::Protocol::V4::LANES[lane]
+        return true unless lane_map
+
+        @policy.allowed!(role:, lane:, key:)
+      end
+
+      def error(code, message)
+        { error: { code: code.to_s, message: } }
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/command_context.rb

@@ -0,0 +1,19 @@
+module Textus
+  module Protocol
+    module Handlers
+      class CommandContext < QueryContext
+        # StoreEngine delegation (write side)
+        def put(...)    = @store_engine.put(...)
+        def delete(...) = @store_engine.delete(...)
+        def move(...)   = @store_engine.move(...)
+
+        # Infrastructure (ops, drain, pulse)
+        def audit_log         = @container.store.audit_log
+        def workflow_queue    = @container.store.workflow_queue
+        def file_system       = @container.store.file_system
+        def workflows         = @container.workflows
+        def workflow_registry = @container.workflow_registry
+      end
+    end
+  end
+end