textus 0.55.2 → 0.56.0

data/lib/textus/protocol/format/script.rb

@@ -0,0 +1,55 @@
+module Textus
+  module Protocol
+    module Format
+      class Script < Base
+        EXTENSIONS = %w[.rb .py .sh .js .pl .rs .exs].freeze
+
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          { "_meta" => {}, "body" => raw, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = meta
+          _ = content
+          b = body.to_s
+          b += "\n" unless b.empty? || b.end_with?("\n")
+          b
+        end
+
+        def self.extensions = EXTENSIONS
+        def self.nested_glob = "**/*"
+        def self.nested_ext(*) = ""
+        def self.enforce_name_match!(_path, _meta); end
+
+        def self.rewrite_name(_path, _basename)
+          false
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested script path must not have an extension")
+          end
+
+          return if EXTENSIONS.include?(ext) || ext == ""
+
+          raise UsageError.new("script format requires #{EXTENSIONS.join(" | ")} extension (got #{ext.inspect})")
+        end
+
+        def self.validate_raw_entry!(_parsed, lane)
+          nil unless lane == "scratchpad"
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/text.rb

@@ -0,0 +1,48 @@
+module Textus
+  module Protocol
+    module Format
+      class Text < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          { "_meta" => {}, "body" => raw, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = meta
+          _ = content
+          b = body.to_s
+          b += "\n" unless b.empty? || b.end_with?("\n")
+          b
+        end
+
+        def self.extensions = [".txt"]
+        def self.nested_glob = "**/*.txt"
+        def self.enforce_name_match!(_path, _meta); end
+
+        def self.rewrite_name(_path, _basename) # rubocop:disable Naming/PredicateMethod
+          false
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested text path must not have an extension")
+          end
+
+          return if [".txt", ""].include?(ext)
+
+          raise UsageError.new("text format requires '.txt' or no extension (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/yaml.rb

@@ -0,0 +1,105 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    module Format
+      class Yaml < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          begin
+            parsed = ::YAML.safe_load(raw, permitted_classes: [Date, Time], aliases: false)
+          rescue Psych::SyntaxError, Psych::AliasesNotEnabled, Psych::DisallowedClass => e
+            raise BadFrontmatter.new(path, "YAML parse failed: #{e.message}")
+          end
+          raise BadFrontmatter.new(path, "YAML top-level must be a mapping") unless parsed.is_a?(Hash)
+
+          meta = parsed["_meta"]
+          fm = meta.is_a?(Hash) ? meta : {}
+          content_without_meta = parsed.except("_meta")
+          { "_meta" => fm, "body" => raw, "content" => content_without_meta }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          if content.is_a?(Hash)
+            on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+            ::YAML.dump(on_disk).sub(/\A---\n/, "")
+          elsif body && !body.to_s.empty?
+            b = body.to_s
+            b += "\n" unless b.end_with?("\n")
+            b
+          else
+            raise UsageError.new("yaml serialize requires :content or :body")
+          end
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["content"] || {})
+        end
+
+        RAW_REQUIRED = %w[ingested_at content_hash].freeze
+        RAW_SOURCE_KINDS = %w[url file asset].freeze
+
+        def self.validate_raw_entry!(parsed, lane)
+          return unless lane == "raw"
+
+          content = parsed["content"] || {}
+          return if content["superseded_by"]
+
+          missing = RAW_REQUIRED.reject { |f| content[f] }
+          raise Textus::BadContent.new(nil, "raw entry missing required field(s): #{missing.join(", ")}") if missing.any?
+
+          source = content["source"] || {}
+          kind = source["kind"]
+          unless RAW_SOURCE_KINDS.include?(kind)
+            raise Textus::BadContent.new(
+              nil, "raw entry source.kind must be #{RAW_SOURCE_KINDS.join("|")}, got #{kind.inspect}"
+            )
+          end
+
+          return unless kind == "url" && !source["url"]
+
+          raise Textus::BadContent.new(nil, "raw entry with source.kind=url must have source.url")
+        end
+
+        def self.extensions = [".yaml", ".yml"]
+        def self.nested_glob = "**/*.{yaml,yml}"
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          raise UsageError.new("put for yaml requires content: or body:") if content.nil? && (body.nil? || body.to_s.empty?)
+
+          if content.nil?
+            begin
+              parsed = parse(body.to_s, path: path)
+            rescue BadFrontmatter => e
+              raise BadContent.new(path, "bad_content: #{e.message}")
+            end
+            [body.to_s, parsed["_meta"], body.to_s, parsed["content"]]
+          else
+            bytes = serialize(meta: meta, body: "", content: content)
+            [bytes, meta, bytes, content]
+          end
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested yaml path must not have an extension")
+          end
+
+          return if [".yaml", ".yml"].include?(ext)
+
+          raise UsageError.new("yaml format requires '.yaml' or '.yml' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format.rb

@@ -0,0 +1,73 @@
+module Textus
+  module Protocol
+    module Format
+      SEP = "---".freeze
+
+      STRATEGIES = {
+        "markdown" => -> { Protocol::Format::Markdown },
+        "json" => -> { Protocol::Format::Json },
+        "yaml" => -> { Protocol::Format::Yaml },
+        "text" => -> { Protocol::Format::Text },
+        "script" => -> { Protocol::Format::Script },
+        "bash" => -> { Protocol::Format::Bash },
+        "ruby" => -> { Protocol::Format::Ruby },
+        "python" => -> { Protocol::Format::Python },
+        "javascript" => -> { Protocol::Format::Javascript },
+      }.freeze
+
+      @registry = nil
+
+      def self.registry=(reg)
+        @registry = reg
+      end
+
+      def self.registry
+        @registry
+      end
+
+      EXT_TO_FORMAT = {
+        ".md" => "markdown",
+        ".json" => "json",
+        ".yaml" => "yaml",
+        ".yml" => "yaml",
+        ".txt" => "text",
+        ".rb" => "ruby",
+        ".py" => "python",
+        ".sh" => "bash",
+        ".js" => "javascript",
+        ".pl" => "script",
+        ".rs" => "script",
+        ".exs" => "script",
+      }.freeze
+
+      def self.for(format)
+        key = format.to_s
+        return registry.fetch(key).call if registry&.key?(key)
+
+        STRATEGIES.fetch(key) { raise Textus::UsageError.new("unknown entry format: #{format.inspect}") }.call
+      end
+
+      def self.infer_from_extension(ext)
+        EXT_TO_FORMAT[ext]
+      end
+
+      def self.formats
+        EXT_TO_FORMAT.values.uniq
+      end
+
+      def self.parse(raw, path: nil, format: "markdown")
+        Protocol::Format.for(format).parse(raw, path: path)
+      end
+
+      def self.serialize(meta: {}, body: "", content: nil, format: "markdown")
+        Protocol::Format.for(format).serialize(meta: meta, body: body, content: content)
+      end
+
+      def self.data_to_payload(data, format)
+        return { meta: {}, body: "", content: nil } if data.nil?
+
+        Protocol::Format.for(format).data_to_payload(data)
+      end
+    end
+  end
+end

data/lib/textus/{store → protocol}/freshness/ttl_evaluator.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Textus
-  class Store
+  module Protocol
     module Freshness
       class TtlEvaluator
         def initialize(manifest:, file_stat:, clock:)
@@ -11,7 +11,7 @@ module Textus
         end
 
         def verdict(mentry)
-          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
+          ttl = lane_ttl(mentry.lane)
           return fresh if ttl.nil?
 
           stale = age_stale?(file_basis(mentry), ttl)
@@ -33,11 +33,15 @@ module Textus
           @file_stat.mtime(path)
         end
 
+        def lane_ttl(lane)
+          Textus::Protocol::V4::LANES.dig(lane, :retention_ttl)
+        end
+
         def due?(mentry, prefix:, lane:)
           return false if lane && mentry.lane != lane
           return false if prefix && !mentry.key.start_with?(prefix)
 
-          ttl = @manifest.rules.for(mentry.key).retention&.ttl_seconds
+          ttl = lane_ttl(mentry.lane)
           return false if ttl.nil?
 
           path = @manifest.resolver.resolve(mentry.key).path

data/lib/textus/protocol/gate.rb

@@ -0,0 +1,61 @@
+module Textus
+  module Protocol
+    class Gate
+      HANDLER_GROUPS = {
+        read: Handlers::Read,
+        write: Handlers::Write,
+        maintenance: Handlers::System,
+      }.freeze
+
+      def initialize(manifest:, resolver:, container:, store_engine:, policy: nil, async: nil)
+        @manifest = manifest
+        @resolver = resolver
+        @container = container
+        @store_engine = store_engine
+        @async = async
+        @policy = policy || manifest.policy
+      end
+
+      def dispatch(role, verb, params = {}, call: nil)
+        spec = VerbRegistry.for(verb)
+        return error(:unknown_verb, "Unknown verb: #{verb}") unless spec
+
+        key = params[:key]
+        authorize!(role, verb, key, spec)
+
+        call_obj = call || Value::Call.build(role:, correlation_id: SecureRandom.uuid)
+        bound = Protocol::Binder.bind(spec, params)
+        ctx_class = spec.read? && !spec.system? ? Handlers::QueryContext : Handlers::CommandContext
+        ctx = ctx_class.new(@store_engine, @container, call_obj)
+
+        handler = spec.lane ? Lane.handler_for(spec.lane) : handler_group(spec)
+        handler.public_send(verb, **bound, ctx:, call: call_obj)
+      end
+
+      private
+
+      def handler_group(spec)
+        return Handlers::System if spec.system?
+
+        HANDLER_GROUPS[spec.category] || Handlers::System
+      end
+
+      def authorize!(role, _verb, key, spec)
+        return true unless spec&.write?
+
+        key_lane = key&.split(".")&.first
+        lane = key_lane || spec.lane&.to_s
+        return true unless lane
+
+        lane_map = Textus::Protocol::V4::LANES[lane]
+        return true unless lane_map
+
+        @policy.allowed!(role:, lane:, key:)
+      end
+
+      def error(code, message)
+        { error: { code: code.to_s, message: } }
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/command_context.rb

@@ -0,0 +1,19 @@
+module Textus
+  module Protocol
+    module Handlers
+      class CommandContext < QueryContext
+        # StoreEngine delegation (write side)
+        def put(...)    = @store_engine.put(...)
+        def delete(...) = @store_engine.delete(...)
+        def move(...)   = @store_engine.move(...)
+
+        # Infrastructure (ops, drain, pulse)
+        def audit_log         = @container.store.audit_log
+        def workflow_queue    = @container.store.workflow_queue
+        def file_system       = @container.store.file_system
+        def workflows         = @container.workflows
+        def workflow_registry = @container.workflow_registry
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/concern.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Protocol
+    module Handlers
+      module Concern
+        def self.included(base)
+          base.extend(self)
+        end
+
+        def response(**fields)
+          { "protocol" => Textus::PROTOCOL, **fields }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/audit_filter.rb

@@ -0,0 +1,37 @@
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class AuditFilter
+          FILTER_KEYS = %i[key lane role verb since seq_since correlation_id limit].freeze
+
+          def call(ctx:, filters: {})
+            seq = filters[:seq_since]
+            if seq
+              min = ctx.audit_log.min_available
+              raise Textus::CursorExpired.new(requested: seq, min_available: min) if min && seq < min - 1
+            end
+
+            rows = ctx.audit_log.list(seq_since: filters[:seq_since], key: filters[:key], role: filters[:role], verb: filters[:verb],
+                                      correlation_id: filters[:correlation_id], limit: filters[:limit]).select do |row|
+              next false if filters[:lane] && !key_in_lane?(row["key"], filters[:lane], ctx)
+              next false if filters[:since] && row["ts"] && Time.parse(row["ts"]) < filters[:since]
+
+              true
+            end
+            { "verb" => "audit", "rows" => rows }
+          end
+
+          private
+
+          def key_in_lane?(key, lane_name, ctx)
+            mentry = ctx.mentry_for(key:)
+            mentry && mentry.lane == lane_name
+          rescue Textus::Error
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/bulk_planner.rb

@@ -0,0 +1,38 @@
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class BulkPlanner
+          Plan = Data.define(:steps, :warnings) do
+            def to_h
+              { "steps" => steps, "warnings" => warnings }
+            end
+          end
+
+          def plan_mv(from_prefix:, to_prefix:, ctx:)
+            raise Textus::UsageError.new("from_prefix and to_prefix required") unless from_prefix && to_prefix
+
+            leaves = ctx.list(prefix: from_prefix)
+            raise Textus::UsageError.new("from_prefix is a leaf — use mv") if leaves.any? { |r| r["key"] == from_prefix }
+
+            warnings = leaves.empty? ? ["no keys under #{from_prefix}"] : []
+            steps = leaves.map do |row|
+              tail = row["key"].delete_prefix("#{from_prefix}.")
+              { "op" => "mv", "from" => row["key"], "to" => "#{to_prefix}.#{tail}" }
+            end
+            Plan.new(steps:, warnings:)
+          end
+
+          def plan_delete(prefix:, ctx:)
+            raise Textus::UsageError.new("prefix required") unless prefix && !prefix.empty?
+
+            leaves = ctx.list(prefix:)
+            warnings = leaves.empty? ? ["no keys under #{prefix}"] : []
+            steps = leaves.map { |row| { "op" => "delete", "key" => row["key"] } }
+            Plan.new(steps:, warnings:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/data_mv.rb

@@ -0,0 +1,49 @@
+require "fileutils"
+
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class DataMv
+          Plan = Data.define(:steps, :warnings) do
+            def to_h
+              { "steps" => steps, "warnings" => warnings }
+            end
+          end
+
+          def call(from:, to:, ctx:, dry_run: false)
+            raise Textus::UsageError.new("from and to required") unless from && to
+            raise Textus::UsageError.new("data lane '#{from}' not declared") unless ctx.manifest.data.declared_lane_kinds.key?(from)
+
+            dest_dir = ctx.layout.lane_path(to)
+            raise Textus::UsageError.new("destination data/#{to} already exists") if File.exist?(dest_dir)
+
+            affected_keys = ctx.manifest.data.entries.select { |e| e.lane == from }.map(&:key)
+            steps = [{ "op" => "rename_zone", "from" => from, "to" => to }]
+            steps += affected_keys.map { |k| { "op" => "mv", "from" => k, "to" => "#{to}#{k[from.length..]}" } }
+            plan = Plan.new(steps:, warnings: [])
+            return plan.to_h if dry_run
+
+            rewrite_manifest!(from:, to:, ctx:)
+            FileUtils.mv(ctx.layout.lane_path(from), dest_dir)
+            plan.to_h
+          end
+
+          private
+
+          def rewrite_manifest!(from:, to:, ctx:)
+            path = ctx.layout.manifest_path
+            raw = YAML.safe_load_file(path, permitted_classes: [Symbol], aliases: false)
+            raw["lanes"].each { |lane| lane["name"] = to if lane["name"] == from }
+            raw["entries"].each do |entry|
+              entry["lane"] = to if entry["lane"] == from
+              entry["key"] = entry["key"].sub(/\A#{Regexp.escape(from)}(\.|\z)/, "#{to}\\1")
+              entry["path"] = entry["path"].sub(%r{\A(data/)?#{Regexp.escape(from)}(/|\z)}, "\\1#{to}\\2")
+            end
+            File.write(path, YAML.dump(raw))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/git_blame.rb

@@ -0,0 +1,65 @@
+require "open3"
+
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class GitBlame
+          def call(key:, ctx:)
+            root = ctx.manifest.data.root
+            path = resolve_blame_path(key, ctx)
+            return nil unless path && git_tracked?(path, root:)
+
+            audit_rows = ctx.audit_log.list(key:)
+            audit_rows.to_h { |row| [row["ts"], git_commit_at(path, timestamp: row["ts"], root:)] }
+          end
+
+          private
+
+          def resolve_blame_path(key, ctx)
+            res = ctx.resolve(key:)
+            res.path || Textus::Protocol::Key::Path.resolve(ctx.manifest.data, res.entry)
+          rescue Textus::Error
+            nil
+          end
+
+          def git_tracked?(path, root:)
+            return false if path.nil? || !File.exist?(path) || !git_repo?(root)
+
+            _out, _err, status = Open3.capture3("git", "ls-files", "--error-unmatch", path, chdir: root)
+            status.success?
+          rescue Errno::ENOENT
+            false
+          end
+
+          def git_repo?(root)
+            dir = root
+            loop do
+              return true if File.directory?(File.join(dir, ".git"))
+
+              parent = File.dirname(dir)
+              return false if parent == dir
+
+              dir = parent
+            end
+          end
+
+          def git_commit_at(path, timestamp:, root:)
+            args = ["git", "log", "-1"]
+            args << "--before=#{timestamp}" if timestamp
+            args += ["--format=%H%x09%an%x09%aI%x09%s", "--", path]
+            out, _err, status = Open3.capture3(*args, chdir: root)
+            return nil unless status.success?
+
+            sha, author, date, subject = out.strip.split("\t", 4)
+            return nil if sha.nil? || sha.empty?
+
+            { "sha" => sha, "author" => author, "date" => date, "subject" => subject }
+          rescue Errno::ENOENT
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/pulse_coordinator.rb

@@ -0,0 +1,37 @@
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class PulseCoordinator
+          def call(role:, ctx:, since: nil)
+            cursor_val = since || ctx.cursor_store.get(role)[:seq]
+            audit_filter = AuditFilter.new
+            audit_result = audit_filter.call(ctx:, filters: { seq_since: cursor_val })
+            changed = audit_result["rows"] || []
+            queue = ctx.manifest.policy.queue_lane
+            pending = queue ? ctx.list(lane: queue) : []
+            runner = Textus::Lanes::Knowledge::Runner.new(ctx.manifest, cursor_store: ctx.cursor_store)
+            result = {
+              "cursor" => ctx.audit_log.latest[:seq],
+              "changed" => changed,
+              "pending_review" => pending.is_a?(Array) ? pending.map { |r| r["key"] } : [],
+              "contract_etag" => Textus::Value::Etag.for_contract(ctx.layout),
+              "index_etag" => pulse_index_etag(ctx),
+              "loop_state" => runner.current_position(role),
+            }
+            ctx.cursor_store.set(role, result["cursor"])
+            result
+          end
+
+          private
+
+          def pulse_index_etag(ctx)
+            ctx.etag_for("artifacts.system.index")
+          rescue Textus::Error
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/handlers/ops/source_expander.rb

@@ -0,0 +1,41 @@
+module Textus
+  module Protocol
+    module Handlers
+      module Ops
+        class SourceExpander
+          def call(env, ctx:, depth: 0)
+            return env if depth >= 5
+
+            raw_sources = Array(env.meta["sources"])
+            return env if raw_sources.empty?
+
+            expanded = raw_sources.map { |src| expand_one(src, ctx:, depth:) }
+            env.with(sources: expanded)
+          end
+
+          private
+
+          def expand_one(src, ctx:, depth:)
+            src = { "key" => src } if src.is_a?(String)
+            return src unless src.is_a?(Hash) && src["key"].is_a?(String)
+
+            key = src["key"]
+            stored_etag = src["etag"]
+            res = ctx.resolve(key:)
+            path = res.path
+            current_etag = File.exist?(path) ? ctx.file_system.etag(path) : nil
+            suspended = stored_etag && current_etag ? stored_etag != current_etag : false
+            result = src.merge("suspended" => suspended)
+            child_env = ctx.read(key:)
+            if child_env
+              child_expanded = call(child_env, ctx:, depth: depth + 1)
+              child_sources = Array(child_expanded.sources)
+              result = result.merge("sources" => child_sources) unless child_sources.empty?
+            end
+            result
+          end
+        end
+      end
+    end
+  end
+end