textus 0.54.2 → 0.55.2

data/lib/textus/doctor/check/schemas.rb

@@ -7,7 +7,7 @@ module Textus
           manifest.data.entries.each do |entry|
             next if entry.schema.nil?
 
-            sp = File.join(root, "schemas", "#{entry.schema}.yaml")
+            sp = geometry.schema_path(entry.schema)
             next if File.exist?(sp)
 
             out << {

data/lib/textus/doctor/check/{notebook_sources.rb → scratchpad_sources.rb}

@@ -1,11 +1,11 @@
 module Textus
   module Doctor
     class Check
-      class NotebookSources < Check
+      class ScratchpadSources < Check
         def call
           issues = []
           manifest.resolver.enumerate.each do |row|
-            next unless row[:key].start_with?("notebook.notes.")
+            next unless row[:key].start_with?("scratchpad.notes.")
             next unless row[:path] && File.exist?(row[:path])
 
             sources = parse_sources(row[:path])
@@ -13,10 +13,10 @@ module Textus
               next if raw_entry_exists?(raw_key)
 
               issues << {
-                "code" => "notebook.source_missing",
+                "code" => "scratchpad.source_missing",
                 "level" => "warning",
                 "subject" => row[:key],
-                "message" => "notebook entry '#{row[:key]}' references raw key '#{raw_key}' " \
+                "message" => "scratchpad entry '#{row[:key]}' references raw key '#{raw_key}' " \
                              "which does not exist in the store",
                 "fix" => "re-ingest the source: textus ingest ..., or remove the stale sources: entry",
               }
@@ -33,7 +33,12 @@ module Textus
           return [] unless match
 
           front = YAML.safe_load(match[1])
-          Array(front&.dig("sources"))
+          Array(front&.dig("sources")).filter_map do |s|
+            case s
+            when String then s
+            when Hash   then s["key"]
+            end
+          end
         rescue StandardError
           []
         end

data/lib/textus/doctor/check/sentinels.rb

@@ -3,13 +3,13 @@ module Textus
     class Check
       class Sentinels < Check
         def call
-          store      = Textus::Ports::SentinelStore.new
-          file_stat  = Textus::Ports::Storage::FileStat.new
-          dir        = Textus::Layout.sentinels(root)
+          store      = Textus::Port::SentinelStore.new
+          file_stat  = Textus::Port::Storage::FileStat.new
+          dir        = Textus::Store::Layout.new(root).sentinels_root
           return [] unless file_stat.directory?(dir)
 
           repo_root = File.dirname(root)
-          file_stat.glob(File.join(dir, "**", "*#{Textus::Ports::SentinelStore::SUFFIX}")).flat_map do |sentinel_path|
+          file_stat.glob(File.join(dir, "**", "*#{Textus::Port::SentinelStore::SUFFIX}")).flat_map do |sentinel_path|
             inspect_sentinel(sentinel_path, repo_root, store, file_stat)
           end
         end

data/lib/textus/doctor/check/templates.rb

@@ -9,7 +9,7 @@ module Textus
               template = target.template
               next if template.nil?
 
-              tp = File.join(root, "templates", template)
+              tp = geometry.template_path(template)
               next if File.exist?(tp)
 
               out << {

data/lib/textus/doctor/check/unowned_schema_fields.rb

@@ -3,7 +3,7 @@ module Textus
     class Check
       class UnownedSchemaFields < Check
         def call
-          dir = File.join(root, "schemas")
+          dir = geometry.schemas_dir
           return [] unless File.directory?(dir)
 
           Dir.glob(File.join(dir, "*.yaml")).flat_map do |path|

data/lib/textus/doctor/check.rb

@@ -14,7 +14,7 @@ module Textus
                           .downcase
       end
 
-      def initialize(container, role: Textus::Role::DEFAULT)
+      def initialize(container, role: Textus::Value::Role::DEFAULT)
         @container = container
         @role      = role
       end
@@ -26,18 +26,17 @@ module Textus
       protected
 
       def root     = @container.root
+      def geometry = @container.layout
       def manifest = @container.manifest
 
-      # Dispatch a verb through Gate.
       def dispatch(verb, *args, **kwargs)
-        klass = Textus::Action::VERBS[verb]
-        spec = klass.contract if klass.respond_to?(:contract?) && klass.contract?
-        inputs = spec ? Textus::Contract::Binder.inputs_from_ordered(spec, args, kwargs) : kwargs
-        cmd_class = Textus::Gate::VERB_COMMAND.fetch(verb)
-        merged = inputs.merge(role: @role)
-        filled = cmd_class.members.to_h { |m| [m, merged.key?(m) ? merged[m] : nil] }
-        cmd = cmd_class.new(**filled)
-        @container.gate.dispatch(cmd)
+        contract_class = Textus::VerbRegistry.contract_class_for(verb)
+        members = contract_class.members
+        command_kwargs = members.each_with_index.to_h { |m, i| [m, args[i] || kwargs[m]] }
+        command = contract_class.new(**command_kwargs)
+        call = Textus::Value::Call.build(role: @role)
+        result = @container.pipeline.dispatch(command, call: call)
+        Textus::Value::Result.extract(result)
       end
     end
   end

data/lib/textus/doctor.rb

@@ -22,7 +22,7 @@ module Textus
       Check::ProposalTargets,
       Check::GeneratorDrift,
       Check::RawAssetPaths,
-      Check::NotebookSources,
+      Check::ScratchpadSources,
       Check::StaleReviewedStamp,
     ].freeze
 
@@ -30,7 +30,7 @@ module Textus
 
     module_function
 
-    def build(container:, checks: nil, role: Textus::Role::DEFAULT)
+    def build(container:, checks: nil, role: Textus::Value::Role::DEFAULT)
       selected_keys = checks ? Array(checks).map(&:to_s) : ALL_CHECKS
       unknown = selected_keys - ALL_CHECKS
       unless unknown.empty?

data/lib/textus/errors.rb

@@ -225,7 +225,15 @@ module Textus
     end
   end
 
+  class ActionError < Error
+    def initialize(code, message, details: {})
+      super(code, message, details: details, exit_code: 1)
+    end
+  end
+
   class CursorExpired < Error
+    JSONRPC_CODE = -32_002
+
     attr_reader :requested, :min_available
 
     def initialize(requested:, min_available:)

data/lib/textus/format/base.rb

@@ -25,18 +25,46 @@ module Textus
         raise NotImplementedError.new("#{name}.nested_glob not implemented")
       end
 
-      def self.validate_path_extension(_path, _nested)
-        raise NotImplementedError.new("#{name}.validate_path_extension not implemented")
+      def self.validate_path_extension(path, nested)
+        ext = File.extname(path)
+        if nested
+          return if ext == ""
+
+          raise UsageError.new("#{format_name} nested path must not have an extension")
+        end
+
+        return if extensions.include?(ext)
+
+        raise UsageError.new("#{format_name} format requires '#{extensions.join("' or '")}' path (got #{ext.inspect})")
       end
 
-      def self.inject_uid(_meta, _content, _existing_uid)
-        raise NotImplementedError.new("#{name}.inject_uid not implemented")
+      def self.enforce_name_match!(path, meta)
+        return unless meta.is_a?(Hash) && meta["name"]
+
+        ext = extensions.first
+        basename = File.basename(path, ext)
+        return if meta["name"] == basename
+
+        raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
       end
 
-      def self.enforce_name_match!(_path, _meta)
-        raise NotImplementedError.new("#{name}.enforce_name_match! not implemented")
+      def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
+        raw = File.binread(path)
+        parsed = parse(raw, path: path)
+        meta = parsed["_meta"] || {}
+        return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
+
+        new_meta = meta.merge("name" => basename)
+        File.binwrite(path, serialize(meta: new_meta, body: parsed["body"] || "", content: parsed["content"]))
+        true
       end
 
+      def self.format_name
+        name.split("::").last.downcase
+      end
+
+      def self.validate_raw_entry!(_parsed, _lane); end
+
       def self.serialize_for_put(meta:, body:, content:, path:)
         _ = meta
         _ = body
@@ -44,10 +72,6 @@ module Textus
         _ = path
         raise NotImplementedError.new("#{name}.serialize_for_put not implemented")
       end
-
-      def self.rewrite_name(_path, _basename)
-        raise NotImplementedError.new("#{name}.rewrite_name not implemented")
-      end
     end
   end
 end

data/lib/textus/format/json.rb

@@ -59,33 +59,6 @@ module Textus
         end
       end
 
-      def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
-        raw = File.binread(path)
-        parsed = parse(raw, path: path)
-        meta = parsed["_meta"]
-        return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
-
-        new_meta = meta.merge("name" => basename)
-        File.binwrite(path, serialize(meta: new_meta, body: "", content: parsed["content"]))
-        true
-      end
-
-      def self.enforce_name_match!(path, meta)
-        return unless meta.is_a?(Hash) && meta["name"]
-
-        ext = extensions.first
-        basename = File.basename(path, ext)
-        return if meta["name"] == basename
-
-        raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
-      end
-
-      def self.inject_uid(meta, content, existing_uid)
-        m = meta.is_a?(Hash) ? meta.dup : {}
-        m["uid"] = existing_uid || Textus::Uid.mint unless m["uid"].is_a?(String) && !m["uid"].empty?
-        [m, content]
-      end
-
       def self.validate_path_extension(path, nested)
         ext = File.extname(path)
         if nested
@@ -98,6 +71,11 @@ module Textus
 
         raise UsageError.new("json format requires '.json' path (got #{ext.inspect})")
       end
+
+      def self.data_to_payload(data)
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+      end
     end
   end
 end

data/lib/textus/format/markdown.rb

@@ -43,39 +43,17 @@ module Textus
         [bytes, meta, body.to_s, nil]
       end
 
-      def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
-        raw = File.binread(path)
-        parsed = parse(raw, path: path)
-        meta = parsed["_meta"] || {}
-        return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
-
-        new_meta = meta.merge("name" => basename)
-        File.binwrite(path, serialize(meta: new_meta, body: parsed["body"]))
-        true
-      end
-
-      def self.enforce_name_match!(path, meta)
-        return unless meta.is_a?(Hash) && meta["name"]
-
-        ext = extensions.first
-        basename = File.basename(path, ext)
-        return if meta["name"] == basename
-
-        raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
-      end
-
-      def self.inject_uid(meta, content, existing_uid)
-        m = meta.is_a?(Hash) ? meta.dup : {}
-        m["uid"] = existing_uid || Textus::Uid.mint unless m["uid"].is_a?(String) && !m["uid"].empty?
-        [m, content]
-      end
-
       def self.validate_path_extension(path, _nested)
         ext = File.extname(path)
         return if ["", ".md"].include?(ext)
 
         raise UsageError.new("markdown format requires '.md' path (got #{ext.inspect})")
       end
+
+      def self.data_to_payload(data)
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+      end
     end
   end
 end

data/lib/textus/format/text.rb

@@ -20,10 +20,6 @@ module Textus
 
       def self.nested_glob = "**/*.txt"
 
-      def self.inject_uid(meta, content, _existing_uid)
-        [meta, content]
-      end
-
       def self.enforce_name_match!(_path, _meta); end
 
       def self.serialize_for_put(meta:, body:, content:, path:)
@@ -49,6 +45,11 @@ module Textus
 
         raise UsageError.new("text format requires '.txt' or no extension (got #{ext.inspect})")
       end
+
+      def self.data_to_payload(data)
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+      end
     end
   end
 end

data/lib/textus/format/yaml.rb

@@ -37,6 +37,31 @@ module Textus
         schema.validate!(parsed["content"] || {})
       end
 
+      RAW_REQUIRED = %w[ingested_at content_hash].freeze
+      RAW_SOURCE_KINDS = %w[url file asset].freeze
+
+      def self.validate_raw_entry!(parsed, lane)
+        return unless lane == "raw"
+
+        content = parsed["content"] || {}
+        return if content["superseded_by"]
+
+        missing = RAW_REQUIRED.reject { |f| content[f] }
+        raise Textus::BadContent.new(nil, "raw entry missing required field(s): #{missing.join(", ")}") if missing.any?
+
+        source = content["source"] || {}
+        kind = source["kind"]
+        unless RAW_SOURCE_KINDS.include?(kind)
+          raise Textus::BadContent.new(
+            nil, "raw entry source.kind must be #{RAW_SOURCE_KINDS.join("|")}, got #{kind.inspect}"
+          )
+        end
+
+        return unless kind == "url" && !source["url"]
+
+        raise Textus::BadContent.new(nil, "raw entry with source.kind=url must have source.url")
+      end
+
       def self.extensions = [".yaml", ".yml"]
 
       def self.nested_glob = "**/*.{yaml,yml}"
@@ -57,33 +82,6 @@ module Textus
         end
       end
 
-      def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
-        raw = File.binread(path)
-        parsed = parse(raw, path: path)
-        meta = parsed["_meta"]
-        return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
-
-        new_meta = meta.merge("name" => basename)
-        File.binwrite(path, serialize(meta: new_meta, body: "", content: parsed["content"]))
-        true
-      end
-
-      def self.enforce_name_match!(path, meta)
-        return unless meta.is_a?(Hash) && meta["name"]
-
-        ext = extensions.first
-        basename = File.basename(path, ext)
-        return if meta["name"] == basename
-
-        raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
-      end
-
-      def self.inject_uid(meta, content, existing_uid)
-        m = meta.is_a?(Hash) ? meta.dup : {}
-        m["uid"] = existing_uid || Textus::Uid.mint unless m["uid"].is_a?(String) && !m["uid"].empty?
-        [m, content]
-      end
-
       def self.validate_path_extension(path, nested)
         ext = File.extname(path)
         if nested
@@ -96,6 +94,11 @@ module Textus
 
         raise UsageError.new("yaml format requires '.yaml' or '.yml' path (got #{ext.inspect})")
       end
+
+      def self.data_to_payload(data)
+        data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+        { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+      end
     end
   end
 end

data/lib/textus/format.rb

@@ -9,6 +9,18 @@ module Textus
       "text" => -> { Format::Text },
     }.freeze
 
+    # Optional registry for injectable format strategies. Tests or app
+    # initializers can set Format.registry = { "custom" => -> { MyFormat }}
+    @registry = nil
+
+    def self.registry=(reg)
+      @registry = reg
+    end
+
+    def self.registry
+      @registry
+    end
+
     EXT_TO_FORMAT = {
       ".md" => "markdown",
       ".json" => "json",
@@ -18,7 +30,10 @@ module Textus
     }.freeze
 
     def self.for(format)
-      STRATEGIES.fetch(format.to_s) { raise Textus::UsageError.new("unknown entry format: #{format.inspect}") }.call
+      key = format.to_s
+      return registry.fetch(key).call if registry&.key?(key)
+
+      STRATEGIES.fetch(key) { raise Textus::UsageError.new("unknown entry format: #{format.inspect}") }.call
     end
 
     def self.infer_from_extension(ext)
@@ -36,5 +51,11 @@ module Textus
     def self.serialize(meta: {}, body: "", content: nil, format: "markdown")
       Format.for(format).serialize(meta: meta, body: body, content: content)
     end
+
+    def self.data_to_payload(data, format)
+      return { meta: {}, body: "", content: nil } if data.nil?
+
+      Format.for(format).data_to_payload(data)
+    end
   end
 end

data/lib/textus/handlers/maintenance/boot_store.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class BootStore
+        def initialize(container:)
+          @container = container
+        end
+
+        def call(_command, _call)
+          Value::Result.success(Textus::Boot.build(container: @container))
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/doctor_store.rb

@@ -0,0 +1,15 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class DoctorStore
+        def initialize(container:)
+          @container = container
+        end
+
+        def call(command, call)
+          Value::Result.success(Textus::Doctor.build(container: @container, checks: command.checks, role: call.role))
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/drain_store.rb

@@ -0,0 +1,21 @@
+module Textus
+  module Handlers
+    module Maintenance
+      class DrainStore
+        def initialize(container:, job_store:)
+          @container = container
+          @job_store = job_store
+        end
+
+        def call(_command, call)
+          queue = Textus::Store::Jobs::Queue.new(store: @job_store)
+          Textus::Store::Jobs::Planner.seed(container: @container, queue: queue, role: call.role)
+          queue.reclaim(now: Textus::Port::Clock.new.now)
+          summary = Textus::Store::Jobs::Worker.for(container: @container, queue: queue).drain
+          Value::Result.success("protocol" => Textus::PROTOCOL, "ok" => summary.failed.zero?,
+                                "completed" => summary.completed, "failed" => summary.failed)
+        end
+      end
+    end
+  end
+end

data/lib/textus/handlers/maintenance/ingest_entry.rb

@@ -0,0 +1,159 @@
+require "fileutils"
+require "date"
+require "digest"
+
+module Textus
+  module Handlers
+    module Maintenance
+      class IngestEntry
+        SOURCE_KINDS = %w[url file asset].freeze
+        CONTENT_HASH_ALGO = "sha256"
+
+        def initialize(container:)
+          @container = container
+        end
+
+        def call(command, call)
+          unless SOURCE_KINDS.include?(command.kind)
+            return Value::Result.failure(:usage_error,
+                                         "ingest kind must be one of #{SOURCE_KINDS.join("|")}")
+          end
+
+          case command.kind
+          when "url"   then return Value::Result.failure(:usage_error, "ingest url requires url") unless command.url
+          when "file"  then return Value::Result.failure(:usage_error, "ingest file requires path") unless command.path
+          when "asset"
+            return Value::Result.failure(:usage_error, "ingest asset requires path") unless command.path
+            return Value::Result.failure(:usage_error, "ingest asset requires lane") unless command.lane
+          end
+
+          now = Time.now.utc
+          key = derive_key(now, command.kind, command.slug)
+          content_hash = compute_content_hash(command)
+          mentry = @container.manifest.resolver.resolve(key).entry
+          ts = now.iso8601
+
+          structured = build_structured(ts, now, content_hash, command)
+          store = @container.job_store
+          index = Textus::Store::Index::Lookup.new(store:)
+
+          duplicate_key = find_duplicate(index, content_hash, command)
+
+          env = if duplicate_key && duplicate_key != key
+                  supersede_entry(duplicate_key, key, structured, call, store, command)
+                else
+                  write_entry(key, structured, mentry, call)
+                end
+
+          rebuild_index(store)
+          Value::Result.success(env)
+        end
+
+        private
+
+        def derive_key(now, kind, slug)
+          date = now.strftime("%Y.%m.%d")
+          "raw.#{date}.#{kind}-#{slug}"
+        end
+
+        def compute_content_hash(command)
+          digest = Digest::SHA256.new
+          case command.kind
+          when "url" then digest.update(command.url)
+          when "file", "asset" then digest.file(command.path)
+          end
+          "#{CONTENT_HASH_ALGO}:#{digest.hexdigest}"
+        end
+
+        def build_structured(timestamp, now, content_hash, command)
+          base = { "ingested_at" => timestamp, "content_hash" => content_hash }
+          case command.kind
+          when "url"
+            base.merge("source" => { "kind" => "url", "url" => command.url,
+                                     "label" => command.label || command.url }, "body" => nil)
+          when "file"
+            base.merge("source" => { "kind" => "file", "path" => command.path,
+                                     "label" => command.label || File.basename(command.path) },
+                       "body" => File.read(command.path))
+          when "asset"
+            asset_rel = copy_asset(now, command.path, command.lane)
+            base.merge("source" => { "kind" => "asset",
+                                     "label" => command.label || File.basename(command.path) },
+                       "asset" => asset_rel, "body" => nil)
+          end
+        end
+
+        def copy_asset(now, path, lane)
+          date_path = now.strftime("%Y/%m/%d")
+          filename  = File.basename(path)
+          assets_dir = @container.layout.asset_raw_dir(date_path, lane)
+          FileUtils.mkdir_p(assets_dir)
+          FileUtils.cp(path, File.join(assets_dir, filename))
+          sentinel = @container.layout.asset_sentinel_path
+          File.write(sentinel, "*\n") unless File.exist?(sentinel)
+          "raw/#{date_path}/#{lane}/#{filename}"
+        end
+
+        def write_entry(key, structured, mentry, call)
+          writer = Store::Entry::Writer.from(container: @container, call: call)
+          writer.put(key, mentry: mentry,
+                          payload: Textus::Value::Payload.new(meta: nil, body: nil, content: structured))
+        end
+
+        def find_duplicate(index, content_hash, command)
+          dup = index.find_by_hash(content_hash)
+          return dup if dup
+          return unless command.kind == "url"
+
+          index.find_by_url(command.url)
+        end
+
+        def supersede_entry(old_key, new_key, structured, call, store, command)
+          old_mentry = @container.manifest.resolver.resolve(old_key).entry
+          reader = Store::Entry::Reader.from(container: @container)
+          old_env = reader.read(old_key)
+          old_content = old_env&.content || {}
+          tombstone = {}
+          %w[ingested_at].each { |k| tombstone[k] = old_content[k] if old_content.key?(k) }
+          source_kind = old_content.dig("source", "kind")
+          tombstone["source"] = { "kind" => source_kind } if source_kind
+          tombstone["superseded_by"] = new_key
+
+          writer = Store::Entry::Writer.from(container: @container, call: call)
+          writer.put(old_key, mentry: old_mentry,
+                              payload: Textus::Value::Payload.new(meta: nil, body: nil, content: tombstone))
+
+          structured["supersedes"] = old_key
+          env = write_entry(new_key, structured,
+                            @container.manifest.resolver.resolve(new_key).entry, call)
+
+          move_asset(old_content["asset"], command.lane) if command.kind == "asset" && old_content["asset"]
+
+          rebuild_index(store)
+          env
+        end
+
+        def move_asset(old_rel, lane)
+          old_path = @container.layout.asset_resolve(old_rel)
+          return unless File.exist?(old_path)
+
+          now = Time.now.utc
+          date_path = now.strftime("%Y/%m/%d")
+          filename = File.basename(old_path)
+          new_dir = @container.layout.asset_raw_dir(date_path, lane)
+          new_path = File.join(new_dir, filename)
+          return if old_path == new_path
+
+          FileUtils.mkdir_p(new_dir)
+          FileUtils.mv(old_path, new_path)
+        rescue Errno::ENOENT, Errno::EACCES => e
+          warn "[textus ingest] could not move asset #{old_rel}: #{e.message}"
+        end
+
+        def rebuild_index(store)
+          Textus::Store::Index::Builder.new(store:).rebuild!(resolver: @container.manifest.resolver)
+        end
+      end
+    end
+  end
+end