textus 0.55.2 → 0.56.0

data/lib/textus/protocol/schema.rb

@@ -0,0 +1,117 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    class Schema
+      attr_reader :name, :required, :optional, :fields, :raw
+
+      def self.load(path)
+        raw = YAML.safe_load_file(path, permitted_classes: [Symbol, Date, Time], aliases: false)
+        new(raw)
+      end
+
+      def initialize(raw)
+        @raw = raw || {}
+        @name = @raw["name"]
+        @required = Array(@raw["required"])
+        @optional = Array(@raw["optional"])
+        @fields = @raw["fields"] || {}
+      end
+
+      def to_h
+        @raw
+      end
+
+      def maintained_by(field)
+        meta = @fields[field] or return nil
+        meta["maintained_by"]
+      end
+
+      def unowned_fields
+        @fields.each_with_object([]) do |(name, spec), acc|
+          next unless spec.is_a?(Hash)
+
+          acc << name if spec["maintained_by"].nil?
+        end
+      end
+
+      def evolution
+        raw = @raw["evolution"] || {}
+        raw.each_with_object({}) do |(k, v), h|
+          h[k] = v.is_a?(Date) || v.is_a?(Time) ? v.to_s : v
+        end
+      end
+
+      def validate!(frontmatter)
+        missing = @required - frontmatter.keys
+        raise Textus::SchemaViolation.new("missing" => missing) unless missing.empty?
+
+        known = (@required + @optional).uniq
+        frontmatter.each do |k, v|
+          next unless @fields.key?(k)
+
+          check_type!(k, v, @fields[k])
+        end
+
+        warnings = frontmatter.keys - known
+        warnings.map { |w| "unknown field: #{w}" }
+      end
+
+      TYPE_VALIDATORS = {
+        "string" => :validate_string,
+        "number" => :validate_number,
+        "boolean" => :validate_boolean,
+        "enum" => :validate_enum,
+        "array" => :validate_array,
+        "object" => :validate_object,
+      }.freeze
+
+      private
+
+      def check_type!(field, value, spec)
+        type = spec["type"]
+        return unless type
+
+        send(TYPE_VALIDATORS[type], field, value, spec)
+      end
+
+      def validate_string(field, value, spec)
+        bad!(field, "expected string") unless value.is_a?(String)
+        return unless (max = spec["max"]) && value.bytesize > max
+
+        bad!(field, "exceeds max #{max}")
+      end
+
+      def validate_number(field, value, _spec)
+        bad!(field, "expected number") unless value.is_a?(Numeric)
+      end
+
+      def validate_boolean(field, value, _spec)
+        bad!(field, "expected boolean") unless [true, false].include?(value)
+      end
+
+      def validate_enum(field, value, spec)
+        values = Array(spec["values"])
+        bad!(field, "not in enum #{values.inspect}") unless values.include?(value)
+      end
+
+      def validate_array(field, value, spec)
+        bad!(field, "expected array") unless value.is_a?(Array)
+        return unless (items = spec["items"])
+
+        value.each_with_index { |v, i| check_type!("#{field}[#{i}]", v, items) }
+      end
+
+      def validate_object(field, value, spec)
+        bad!(field, "expected object") unless value.is_a?(Hash)
+        return unless (sub = spec["fields"])
+
+        sub.each { |fk, fspec| check_type!("#{field}.#{fk}", value[fk], fspec) if value.key?(fk) }
+      end
+
+      def bad!(field, reason)
+        raise Textus::SchemaViolation.new("field" => field, "reason" => reason)
+      end
+    end
+  end
+end

data/lib/textus/protocol/session/bridge.rb

@@ -0,0 +1,11 @@
+Textus::Protocol::Session::Bridge = Data.define(:container, :gate) do
+  def store         = container.store
+  def manifest      = container.manifest
+  def schemas       = container.schemas
+  def layout        = container.layout
+  def workflows     = container.workflows
+  def root          = container.root
+  def ttl_evaluator = container.ttl_evaluator
+
+  def dispatch(...) = gate.dispatch(...)
+end

data/lib/textus/protocol/session/context.rb

@@ -0,0 +1,5 @@
+Textus::Protocol::Session::Context = Data.define(:role, :correlation_id, :dry_run, :cursor, :contract_etag, :propose_lane) do
+  def call
+    Textus::Value::Call.build(role:, correlation_id:)
+  end
+end

data/lib/textus/protocol/session.rb

@@ -0,0 +1,87 @@
+require "forwardable"
+
+module Textus
+  module Protocol
+    class Session
+      extend Forwardable
+
+      attr_reader :context, :bridge
+
+      DOMAIN_VERBS = (VerbRegistry::ENTRY_VERBS + VerbRegistry::OPS_VERBS).to_set.freeze
+
+      def_delegators :context, :role, :cursor, :correlation_id, :contract_etag, :propose_lane
+      def_delegator  :context, :dry_run, :dry_run?
+      def_delegators :bridge, :root, :container, :store, :manifest, :schemas, :layout, :workflows, :gate, :ttl_evaluator
+
+      def self.open(root, role: Value::Role::DEFAULT, correlation_id: nil, dry_run: false)
+        root = File.expand_path(root)
+        container = Protocol::Builder.new.call(root)
+        gate = container.gate
+        bridge = Session::Bridge.new(container:, gate:)
+        context = Session::Context.new(
+          role: role.to_s,
+          correlation_id: correlation_id || SecureRandom.uuid,
+          dry_run: dry_run,
+          cursor: container.store.audit_log.latest[:seq],
+          contract_etag: Value::Etag.for_contract(bridge.layout),
+          propose_lane: container.manifest.policy.propose_lane_for(role.to_s),
+        )
+        new(bridge:, context:)
+      end
+
+      def initialize(bridge:, context:)
+        @bridge = bridge
+        @context = context
+      end
+
+      def method_missing(name, *args, **kwargs)
+        if DOMAIN_VERBS.include?(name)
+          raise ArgumentError.new("#{name} accepts keyword arguments only") unless args.empty?
+
+          return bridge.dispatch(role.to_sym, name, kwargs, call: context.call)
+        end
+
+        return bridge.public_send(name, *args, **kwargs) if bridge.respond_to?(name)
+
+        super
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        DOMAIN_VERBS.include?(name) || bridge.respond_to?(name) || super
+      end
+
+      def with_correlation_id(cid)
+        self.class.new(bridge:, context: context.with(correlation_id: cid))
+      end
+
+      def with_role(new_role)
+        self.class.new(
+          bridge:,
+          context: context.with(
+            role: new_role.to_s,
+            cursor: container.store.audit_log.latest[:seq],
+            contract_etag: Value::Etag.for_contract(container.layout),
+            propose_lane: manifest.policy.propose_lane_for(new_role.to_s),
+          ),
+        )
+      end
+
+      def advance_cursor(new_cursor)
+        @context = context.with(cursor: new_cursor)
+        self
+      end
+
+      def check_etag!(observed_etag)
+        return if observed_etag == contract_etag
+
+        raise Textus::ContractDrift.new(
+          "contract changed (was #{short_etag(contract_etag)}, now #{short_etag(observed_etag)})",
+        )
+      end
+
+      private
+
+      def short_etag(etag) = etag.to_s.delete_prefix("sha256:")[0, 8]
+    end
+  end
+end

data/lib/textus/protocol/store_engine/delete.rb

@@ -0,0 +1,37 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Delete
+        module_function
+
+        def call(key, deps:, if_etag: nil)
+          path = Pipeline.resolve_path(key, deps)
+          assert_exists!(key, path, deps)
+          etag_before = Pipeline.check_etag(key, path, if_etag, deps.file_system)
+          deps.file_system.delete(path)
+          deps.entry_index.remove(key:)
+          Pipeline.prune_parents(path, deps)
+          append_audit(key, etag_before, deps)
+          Pipeline.emit_event(deps, event_type: "entry.deleted",
+                                    pending_key: key, emit_key: key)
+          nil
+        end
+
+        def assert_exists!(key, path, deps)
+          return if deps.file_system.exists?(path)
+
+          raise Textus::UnknownKey.new(key,
+                                       suggestions: deps.manifest.resolver.suggestions_for(key))
+        end
+
+        def append_audit(key, etag_before, deps)
+          deps.audit_log.append(
+            role: deps.call.role, verb: "key_delete", key:,
+            etag_before:, etag_after: nil,
+            correlation_id: deps.call.correlation_id
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine/move.rb

@@ -0,0 +1,109 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Move
+        MoveResult = Data.define(:from_key, :to_key, :from_path, :to_path, :dry_run, :envelope)
+
+        module_function
+
+        def call(from_key:, to_key:, deps:, if_etag: nil, dry_run: false)
+          validate!(from_key, to_key)
+          from_path, to_path, from_entry, to_entry = resolve_paths(from_key, to_key, deps)
+          assert_source_exists!(from_key, from_path, deps)
+          assert_target_not_exists!(to_path)
+          check_cross_zone!(from_entry, to_entry)
+          ensure_uid!(from_key, from_entry, from_path, deps)
+          etag_before = Pipeline.check_etag(from_key, from_path, if_etag, deps.file_system)
+
+          if dry_run
+            return MoveResult.new(from_key:, to_key:, from_path:, to_path:,
+                                  dry_run: true, envelope: nil)
+          end
+
+          deps.file_system.mv(from_path, to_path)
+          Pipeline.prune_parents(from_path, deps)
+          rewrite_basename!(to_key, to_path, to_entry)
+          etag_after = Value::Etag.for_file(to_path)
+          envelope = deps.reader.read(key: to_key)
+          append_audit(to_key, etag_before, etag_after, deps)
+          reindex_entry(from_key, to_key, to_entry, envelope, deps)
+          Pipeline.emit_event(deps, event_type: "entry.moved",
+                                    pending_key: from_key, emit_key: to_key,
+                                    emit_lane: to_entry.lane,
+                                    pending_keys: [from_key, to_key])
+          MoveResult.new(from_key:, to_key:, from_path:, to_path:,
+                         dry_run: false, envelope:)
+        end
+
+        def validate!(from_key, to_key)
+          Manifest::Data.validate_key!(from_key)
+          Manifest::Data.validate_key!(to_key)
+          raise UsageError.new("old and new keys are identical") if from_key == to_key
+        end
+
+        def resolve_paths(from_key, to_key, deps)
+          from = deps.manifest.resolver.resolve(from_key)
+          to   = deps.manifest.resolver.resolve(to_key)
+          [from.path, to.path, from.entry, to.entry]
+        end
+
+        def assert_source_exists!(key, path, deps)
+          return if deps.file_system.exists?(path)
+
+          raise UnknownKey.new(key, suggestions: deps.manifest.resolver.suggestions_for(key))
+        end
+
+        def assert_target_not_exists!(path)
+          return unless path && File.exist?(path)
+
+          raise UsageError.new("target already exists")
+        end
+
+        def check_cross_zone!(from_entry, to_entry)
+          return if from_entry.lane == to_entry.lane
+
+          raise UsageError.new("cross-zone moves forbidden (#{from_entry.lane} → #{to_entry.lane})")
+        end
+
+        def ensure_uid!(_key, entry, path, deps)
+          return if Infra::Store::Envelope::Meta::NO_META_FORMATS.include?(entry.format)
+          return unless path && deps.file_system.exists?(path)
+
+          raw = deps.file_system.read(path)
+          parsed = Format.for(entry.format).parse(raw, path:)
+          meta = parsed["_meta"] || {}
+          return if meta["uid"].is_a?(String) && !meta["uid"].empty?
+
+          new_meta, = Infra::Store::Envelope::Meta.inject_all(meta, parsed["content"], {},
+                                                              format: entry.format)
+          bytes, = Format.for(entry.format).serialize_for_put(
+            meta: new_meta, body: parsed["body"] || "", content: parsed["content"], path:,
+          )
+          deps.file_system.write(path, bytes)
+        end
+
+        def rewrite_basename!(key, path, entry)
+          basename = key.split(".").last
+          Format.for(entry.format).rewrite_name(path, basename)
+        end
+
+        def append_audit(key, etag_before, etag_after, deps)
+          deps.audit_log.append(
+            role: deps.call.role, verb: "key_mv", key:,
+            etag_before:, etag_after:,
+            correlation_id: deps.call.correlation_id
+          )
+        end
+
+        def reindex_entry(from_key, to_key, to_entry, envelope, deps)
+          deps.entry_index.remove(key: from_key)
+          content = envelope.content ? envelope.content.to_json : envelope.body
+          deps.entry_index.record(
+            key: to_key, lane: to_entry.lane, format: to_entry.format,
+            schema: to_entry.schema, content:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine/pipeline.rb

@@ -0,0 +1,61 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Pipeline
+        WriteDeps = Data.define(:file_system, :manifest, :store, :call, :reader, :layout, :event_emitter, :schemas) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def file_store   = store.file_store
+        end
+
+        DeleteDeps = Data.define(:file_system, :manifest, :store, :call, :layout, :event_emitter) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def workflow_queue = store.workflow_queue
+        end
+
+        MoveDeps = Data.define(:file_system, :manifest, :store, :call, :reader, :layout, :event_emitter) do
+          def audit_log    = store.audit_log
+          def entry_index  = store.entry_index
+          def workflow_queue = store.workflow_queue
+        end
+
+        module_function
+
+        def resolve_path(key, deps)
+          deps.manifest.resolver.resolve(key).path
+        end
+
+        def check_etag(key, path, if_etag, file_system)
+          etag_before = file_system.exists?(path) ? file_system.etag(path) : nil
+          raise Textus::EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
+
+          etag_before
+        end
+
+        def prune_parents(path, deps)
+          floor = deps.layout.lane_floor(path)
+          return unless floor
+
+          dir = File.dirname(path)
+          while dir.start_with?("#{floor}/") && deps.file_system.dir_empty?(dir)
+            deps.file_system.rmdir(dir)
+            dir = File.dirname(dir)
+          end
+        rescue SystemCallError
+          # race: dir may have been emptied by another writer
+        end
+
+        def emit_event(deps, event_type:, pending_key:, emit_key:, emit_lane: nil, pending_keys: [])
+          return unless deps.event_emitter
+          return if deps.store.workflow_queue.pending?(event_type:, key: pending_key)
+          return if pending_keys.any? { |k| deps.store.workflow_queue.pending?(event_type:, key: k) }
+
+          deps.event_emitter.emit(event_type:, key: emit_key, lane: emit_lane || "artifacts")
+        rescue StandardError => e
+          warn "textus: emit_workflow_event(#{event_type}, #{emit_key}) failed: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine/put.rb

@@ -0,0 +1,104 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      module Put
+        module_function
+
+        def call(key, mentry:, path:, payload:, deps:, if_etag: nil, fmt: nil)
+          format = fmt || mentry.format
+
+          validate_input(key, payload, format, mentry, path)
+          validate_schema(payload, mentry, deps)
+
+          existing_env = read_existing(key, mentry, path, format, deps)
+          meta, content = inject_meta(payload, existing_env, format, deps)
+          bytes, eff_meta, eff_body, eff_content = serialize(payload, meta, content, path, format)
+
+          etag_before = Pipeline.check_etag(key, path, if_etag, deps.file_system)
+          deps.file_system.write(path, bytes)
+          envelope = Envelope.build(
+            key:, mentry:, path:,
+            meta: eff_meta, body: eff_body,
+            etag: Value::Etag.for_bytes(bytes),
+            content: eff_content
+          )
+
+          append_audit(key, etag_before, envelope, deps)
+          index_entry(key, mentry, format, eff_body, eff_content, deps)
+          Pipeline.emit_event(deps, event_type: "entry.written",
+                                    pending_key: key, emit_key: key,
+                                    emit_lane: mentry.lane)
+          envelope
+        end
+
+        def validate_input(key, payload, format, mentry, path)
+          EntryConstraint.validate_naming!(key, mentry)
+          fmt = Format.for(format)
+          fmt.enforce_name_match!(path, payload.meta)
+          fmt.validate_raw_entry!(
+            { "_meta" => payload.meta, "content" => payload.content },
+            mentry.lane,
+          )
+        end
+
+        def validate_schema(payload, mentry, deps)
+          EntryConstraint.validate_schema!(payload, mentry, schemas: deps.schemas)
+        end
+
+        def read_existing(key, mentry, path, format, deps)
+          return nil unless path && deps.file_system.exists?(path)
+
+          raw = deps.file_system.read(path)
+          parsed = Format.for(format).parse(raw, path:)
+          Envelope.build(
+            key:, mentry:, path:,
+            meta: parsed["_meta"] || {},
+            body: parsed["body"] || "",
+            content: parsed["content"],
+            etag: Value::Etag.for_file(path)
+          )
+        end
+
+        def inject_meta(payload, existing_env, format, deps)
+          existing_meta = existing_env ? existing_env.meta : {}
+          Infra::Store::Envelope::Meta.inject_all(
+            payload.meta || {}, payload.content, existing_meta,
+            format:,
+            etag_for: method(:etag_for_key).curry.call(deps)
+          )
+        end
+
+        def etag_for_key(deps, key)
+          path = deps.manifest.resolver.resolve(key).path
+          return nil unless deps.file_system.exists?(path)
+
+          Value::Etag.for_file(path)
+        rescue Textus::Error
+          nil
+        end
+
+        def serialize(payload, meta, content, path, format)
+          Format.for(format).serialize_for_put(
+            meta:, body: payload.body, content:, path:,
+          )
+        end
+
+        def append_audit(key, etag_before, envelope, deps)
+          deps.audit_log.append(
+            role: deps.call.role, verb: "put", key:,
+            etag_before:, etag_after: envelope.etag,
+            correlation_id: deps.call.correlation_id
+          )
+        end
+
+        def index_entry(key, mentry, format, eff_body, eff_content, deps)
+          content = eff_content ? eff_content.to_json : eff_body
+          deps.entry_index.record(
+            key:, lane: mentry.lane, format:,
+            schema: mentry.schema, content:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/store_engine.rb

@@ -0,0 +1,86 @@
+module Textus
+  module Protocol
+    class StoreEngine
+      def initialize(file_system:, manifest:, layout:, store:, event_emitter: nil, schemas: nil)
+        @file_system = file_system
+        @manifest = manifest
+        @layout = layout
+        @store = store
+        @event_emitter = event_emitter
+        @schemas = schemas
+      end
+
+      def read(key:)
+        result = resolve(key:)
+        path = result.path
+        return nil unless path && @file_system.exists?(path)
+
+        raw = @file_system.read(path)
+        parsed = Format.for(result.entry.format).parse(raw, path:)
+        Envelope.build(key:, mentry: result.entry, path:,
+                       meta: parsed["_meta"] || {},
+                       body: parsed["body"] || "",
+                       content: parsed["content"],
+                       etag: Value::Etag.for_file(path))
+      end
+
+      def exists?(key:)
+        path = resolve(key:).path
+        path && @file_system.exists?(path)
+      end
+
+      def list(prefix: nil, lane: nil, q: nil, schema: nil)
+        if @store.entry_index && (q || schema)
+          rows = @store.entry_index.search(query: q, schema:, lane:, prefix:)
+          (rows || []).map { |r| { "key" => r["key"], "lane" => r["lane"] } }
+        else
+          rows = @manifest.resolver.enumerate(prefix:)
+          rows = rows.select { |r| r[:manifest_entry].lane == lane } if lane
+          rows.map { |r| { "key" => r[:key], "lane" => r[:manifest_entry].lane } }
+        end
+      end
+
+      def resolve(key:)   = @manifest.resolver.resolve(key)
+      def mentry_for(key:)= resolve(key:).entry
+
+      def put(key:, meta:, body:, content:, call:, if_etag: nil, format: nil)
+        result = resolve(key:)
+        payload = Value::Payload.new(meta: meta || {}, body:, content:)
+        deps = build_deps(reader: nil, call:)
+
+        if format
+          fmt_mod = Textus::Protocol::Format.for(format)
+          path = result.path.sub(/\.[^.]+$/, "") + fmt_mod.nested_ext(result.remaining.last)
+          Put.call(key, mentry: result.entry, path:, payload:, if_etag:, deps:, fmt: format)
+        else
+          Put.call(key, mentry: result.entry, path: result.path, payload:, if_etag:, deps:)
+        end
+      end
+
+      def delete(key:, call:, if_etag: nil)
+        deps = Pipeline::DeleteDeps.new(
+          file_system: @file_system, manifest: @manifest, store: @store,
+          layout: @layout, event_emitter: @event_emitter, call:
+        )
+        Delete.call(key, if_etag:, deps:)
+      end
+
+      def move(from_key:, to_key:, call:, if_etag: nil, dry_run: false)
+        deps = Pipeline::MoveDeps.new(
+          file_system: @file_system, manifest: @manifest, store: @store,
+          layout: @layout, event_emitter: @event_emitter, reader: self, call:
+        )
+        Move.call(from_key:, to_key:, if_etag:, dry_run:, deps:)
+      end
+
+      private
+
+      def build_deps(reader:, call:)
+        Pipeline::WriteDeps.new(
+          file_system: @file_system, manifest: @manifest, store: @store,
+          layout: @layout, event_emitter: @event_emitter, schemas: @schemas, reader:, call:
+        )
+      end
+    end
+  end
+end

data/lib/textus/protocol/v4.rb

@@ -0,0 +1,25 @@
+module Textus
+  module Protocol
+    module V4
+      LANES = {
+        "knowledge" => { writers: %w[human],
+                         desc: "the maintained source of truth about the repo",
+                         retention_ttl: nil },
+        "scratchpad" => { writers: %w[human agent],
+                          desc: "the agent's own durable scratchpad (working memory)",
+                          retention_ttl: nil },
+        "artifacts" => { writers: %w[automation],
+                         desc: "machine-maintained computed outputs",
+                         retention_ttl: 3600 * 24 * 30 },
+        "raw" => { writers: %w[human agent automation],
+                   desc: "ingested external source material (write-once)",
+                   retention_ttl: nil },
+      }.freeze
+
+      NAMING_PATTERNS = {
+        "sequential" => /\A\d{4}-[a-z0-9-]+\.\w+\z/,
+        "dated" => /\A\d{4}-\d{2}-\d{2}-[a-z0-9-]+\.\w+\z/,
+      }.freeze
+    end
+  end
+end

data/lib/textus/protocol/verb_registry/arg_spec.rb

@@ -0,0 +1,19 @@
+module Textus
+  module Protocol
+    module VerbRegistry
+      ArgSpec = Data.define(
+        :name, :type, :required, :positional,
+        :description, :wire_name, :default, :source, :coerce, :cli_default
+      ) do
+        def wire = wire_name || name
+
+        def self.arg(name:, type: String, required: false, positional: false,
+                     description: nil, wire_name: nil,
+                     default: nil, source: nil, coerce: nil, cli_default: nil)
+          new(name:, type:, required:, positional:,
+              description:, wire_name:, default:, source:, coerce:, cli_default:)
+        end
+      end
+    end
+  end
+end