textus 0.3.0 → 0.5.0

data/lib/textus/doctor.rb

@@ -1,25 +1,32 @@
 require "digest"
 require "json"
+require "timeout"
 
 module Textus
   # Health check for a Textus store. Returns a JSON-friendly Hash envelope
   # with an `issues` array and a summary. Each issue is a Hash with
   # `code`, `level`, `subject`, `message`, and optionally `fix`.
-  module Doctor
+  module Doctor # rubocop:disable Metrics/ModuleLength -- 9 built-in checks + extension dispatch
     LEVELS = %w[error warning info].freeze
+    DOCTOR_CHECK_TIMEOUT_SECONDS = 2
+    ALL_CHECKS = %w[
+      manifest_files schemas templates extensions illegal_keys
+      sentinels audit_log unowned_schema_fields schema_violations
+    ].freeze
 
     module_function
 
-    def run(store)
-      issues = []
-      issues.concat(check_manifest_files(store))
-      issues.concat(check_schemas(store))
-      issues.concat(check_templates(store))
-      issues.concat(check_extensions(store))
-      issues.concat(check_illegal_keys(store))
-      issues.concat(check_sentinels(store))
-      issues.concat(check_audit_log(store))
-      issues.concat(check_unowned_schema_fields(store))
+    def run(store, checks: nil)
+      selected = checks ? Array(checks).map(&:to_s) : ALL_CHECKS
+      unknown = selected - ALL_CHECKS
+      unless unknown.empty?
+        raise UsageError.new(
+          "unknown doctor check: #{unknown.first}. Valid checks: #{ALL_CHECKS.join(", ")}",
+        )
+      end
+
+      issues = run_builtin_checks(store, selected)
+      issues.concat(run_registered_checks(store)) # extensions always run
 
       summary = LEVELS.to_h { |l| [l, issues.count { |i| i["level"] == l }] }
       {
@@ -30,6 +37,20 @@ module Textus
       }
     end
 
+    def run_builtin_checks(store, selected)
+      issues = []
+      issues.concat(check_manifest_files(store))        if selected.include?("manifest_files")
+      issues.concat(check_schemas(store))               if selected.include?("schemas")
+      issues.concat(check_templates(store))             if selected.include?("templates")
+      issues.concat(check_extensions(store))            if selected.include?("extensions")
+      issues.concat(check_illegal_keys(store))          if selected.include?("illegal_keys")
+      issues.concat(check_sentinels(store))             if selected.include?("sentinels")
+      issues.concat(check_audit_log(store))             if selected.include?("audit_log")
+      issues.concat(check_unowned_schema_fields(store)) if selected.include?("unowned_schema_fields")
+      issues.concat(check_schema_violations(store))     if selected.include?("schema_violations")
+      issues
+    end
+
     # --- Checks -----------------------------------------------------------
 
     def check_manifest_files(store)
@@ -196,32 +217,30 @@ module Textus
         stripped = line.chomp
         next if stripped.empty?
 
-        # Audit log is TSV, not NDJSON. Treat as malformed if it doesn't have
-        # at least 6 tab-separated fields (timestamp, role, verb, key, etag_before, etag_after).
-        fields = stripped.split("\t")
-        if fields.length < 6
-          out << {
-            "code" => "audit.parse_error",
-            "level" => "warning",
-            "subject" => "#{path}:#{lineno}",
-            "message" => "audit log line #{lineno} has #{fields.length} fields (expected >=6)",
-            "fix" => "inspect #{path} at line #{lineno} and remove the corrupted row",
-          }
-          next
-        end
-
-        extras = fields[6]
-        next if extras.nil? || extras.empty?
+        if stripped.start_with?("{")
+          begin
+            JSON.parse(stripped)
+          rescue JSON::ParserError => e
+            out << {
+              "code" => "audit.parse_error",
+              "level" => "warning",
+              "subject" => "#{path}:#{lineno}",
+              "message" => "audit log line #{lineno} is invalid JSON: #{e.message}",
+              "fix" => "inspect #{path} at line #{lineno} and remove the corrupted row",
+            }
+          end
+        else
+          # Legacy TSV: minimum 6 fields. Removed in 0.6.
+          fields = stripped.split("\t")
+          next if fields.length >= 6
 
-        begin
-          JSON.parse(extras)
-        rescue JSON::ParserError => e
           out << {
             "code" => "audit.parse_error",
             "level" => "warning",
             "subject" => "#{path}:#{lineno}",
-            "message" => "audit log line #{lineno} extras JSON malformed: #{e.message}",
-            "fix" => "inspect #{path} at line #{lineno} and fix the JSON in the last column",
+            "message" => "audit log line #{lineno} has #{fields.length} fields " \
+                         "(expected >=6 for legacy TSV; consider migrating to NDJSON)",
+            "fix" => "inspect #{path} at line #{lineno} and remove the corrupted row",
           }
         end
       end
@@ -255,6 +274,58 @@ module Textus
       out
     end
 
+    def check_schema_violations(store)
+      res = store.validate_all
+      res["violations"].map do |v|
+        fix = v["expected"] &&
+              "field '#{v["field"]}' should be written by '#{v["expected"]}' (last writer: #{v["last_writer"]})"
+        {
+          "code" => v["code"],
+          "level" => "error",
+          "subject" => v["key"],
+          "message" => v["message"] || "#{v["code"]} on #{v["key"]}",
+          "fix" => fix,
+        }.compact
+      end
+    end
+
+    def run_registered_checks(store)
+      out = []
+      view = StoreView.new(store)
+      store.registry.doctor_check_names.each do |name|
+        callable = store.registry.doctor_check(name)
+        begin
+          result = Timeout.timeout(DOCTOR_CHECK_TIMEOUT_SECONDS) { callable.call(store: view) }
+          if result.is_a?(Array)
+            out.concat(result.map { |h| h.transform_keys(&:to_s) })
+          else
+            out << fail_issue(name, code: "doctor_check.bad_return",
+                                    message: "doctor_check '#{name}' returned #{result.class} (expected Array)",
+                                    fix: "return an array of issue hashes from the doctor_check block")
+          end
+        rescue Timeout::Error
+          out << fail_issue(name, code: "doctor_check.timeout",
+                                  message: "doctor_check '#{name}' exceeded #{DOCTOR_CHECK_TIMEOUT_SECONDS}s",
+                                  fix: "shorten the check or split it into smaller checks")
+        rescue StandardError => e
+          out << fail_issue(name, code: "doctor_check.failed",
+                                  message: "#{e.class}: #{e.message}",
+                                  fix: "fix the doctor_check block in .textus/extensions/")
+        end
+      end
+      out
+    end
+
+    def fail_issue(name, code:, message:, fix:)
+      {
+        "code" => code,
+        "level" => "error",
+        "subject" => name.to_s,
+        "message" => message,
+        "fix" => fix,
+      }
+    end
+
     # --- Helpers ----------------------------------------------------------
 
     def leaf_path_for(store, entry)

data/lib/textus/entry/json.rb

@@ -17,13 +17,15 @@ module Textus
 
         meta = parsed["_meta"]
         fm = meta.is_a?(Hash) ? meta : {}
-        { "frontmatter" => fm, "body" => raw, "content" => parsed }
+        content_without_meta = parsed.except("_meta")
+        { "_meta" => fm, "body" => raw, "content" => content_without_meta }
       end
 
-      def self.serialize(frontmatter:, body:, content: nil)
-        _ = frontmatter
+      def self.serialize(meta:, body:, content: nil)
         if content.is_a?(Hash)
-          out = ::JSON.pretty_generate(content)
+          # Re-inject _meta as the first key so on-disk shape is stable.
+          on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+          out = ::JSON.pretty_generate(on_disk)
           out += "\n" unless out.end_with?("\n")
           out
         elsif body && !body.to_s.empty?

data/lib/textus/entry/markdown.rb

@@ -7,7 +7,7 @@ module Textus
       def self.parse(raw, path: nil)
         raw = raw.dup.force_encoding(Encoding::UTF_8)
         raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
-        return { "frontmatter" => {}, "body" => raw, "content" => nil } unless raw.start_with?("---\n") || raw.start_with?("---\r\n")
+        return { "_meta" => {}, "body" => raw, "content" => nil } unless raw.start_with?("---\n") || raw.start_with?("---\r\n")
 
         lines = raw.split(/\r?\n/, -1)
         close_idx = lines[1..].index("---")
@@ -22,12 +22,12 @@ module Textus
           raise BadFrontmatter.new(path, "YAML parse failed: #{e.message}")
         end
         fm = {} unless fm.is_a?(Hash)
-        { "frontmatter" => fm, "body" => body, "content" => nil }
+        { "_meta" => fm, "body" => body, "content" => nil }
       end
 
-      def self.serialize(frontmatter:, body:, content: nil)
+      def self.serialize(meta:, body:, content: nil)
         _ = content # markdown ignores content
-        fm_yaml = frontmatter.empty? ? "" : ::YAML.dump(frontmatter).sub(/\A---\n/, "")
+        fm_yaml = meta.empty? ? "" : ::YAML.dump(meta).sub(/\A---\n/, "")
         body = body.to_s
         body += "\n" unless body.empty? || body.end_with?("\n")
         "---\n#{fm_yaml}---\n#{body}"

data/lib/textus/entry/text.rb

@@ -6,11 +6,11 @@ module Textus
         raw = raw.dup.force_encoding(Encoding::UTF_8)
         raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
 
-        { "frontmatter" => {}, "body" => raw, "content" => nil }
+        { "_meta" => {}, "body" => raw, "content" => nil }
       end
 
-      def self.serialize(frontmatter:, body:, content: nil)
-        _ = frontmatter
+      def self.serialize(meta:, body:, content: nil)
+        _ = meta
         _ = content
         b = body.to_s
         b += "\n" unless b.empty? || b.end_with?("\n")

data/lib/textus/entry/yaml.rb

@@ -17,13 +17,15 @@ module Textus
 
         meta = parsed["_meta"]
         fm = meta.is_a?(Hash) ? meta : {}
-        { "frontmatter" => fm, "body" => raw, "content" => parsed }
+        content_without_meta = parsed.except("_meta")
+        { "_meta" => fm, "body" => raw, "content" => content_without_meta }
       end
 
-      def self.serialize(frontmatter:, body:, content: nil)
-        _ = frontmatter
+      def self.serialize(meta:, body:, content: nil)
         if content.is_a?(Hash)
-          ::YAML.dump(content).sub(/\A---\n/, "")
+          # Re-inject _meta as the first key so on-disk shape is stable.
+          on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+          ::YAML.dump(on_disk).sub(/\A---\n/, "")
         elsif body && !body.to_s.empty?
           b = body.to_s
           b += "\n" unless b.end_with?("\n")

data/lib/textus/entry.rb

@@ -23,8 +23,8 @@ module Textus
       for_format(format).parse(raw, path: path)
     end
 
-    def self.serialize(frontmatter: {}, body: "", content: nil, format: "markdown")
-      for_format(format).serialize(frontmatter: frontmatter, body: body, content: content)
+    def self.serialize(meta: {}, body: "", content: nil, format: "markdown")
+      for_format(format).serialize(meta: meta, body: body, content: content)
     end
   end
 end

data/lib/textus/errors.rb

@@ -51,10 +51,10 @@ module Textus
     private
 
     def default_hint_for(path, m)
-      if m.is_a?(String) && (match = m.match(/frontmatter name '([^']+)' does not match basename '([^']+)'/))
+      if m.is_a?(String) && (match = m.match(/name '([^']+)' does not match basename '([^']+)'/))
         name, basename = match.captures
         ext = File.extname(path)
-        "rename the file to '#{name}#{ext}' or change frontmatter name: to '#{basename}'"
+        "rename the file to '#{name}#{ext}' or change _meta.name to '#{basename}'"
       else
         "open #{path} and check the YAML frontmatter for syntax errors"
       end

data/lib/textus/extension_registry.rb

@@ -3,16 +3,17 @@ module Textus
     EVENTS = %i[put delete refresh build accept].freeze
 
     def initialize
-      @fetchers = {}
+      @actions = {}
       @reducers = {}
       @hooks = {}
+      @doctor_checks = {}
     end
 
-    def register_fetcher(name, &blk)
+    def register_action(name, &blk)
       name = name.to_sym
-      raise UsageError.new("fetcher '#{name}' already registered") if @fetchers.key?(name)
+      raise UsageError.new("action '#{name}' already registered") if @actions.key?(name)
 
-      @fetchers[name] = blk
+      @actions[name] = blk
     end
 
     def register_reducer(name, &blk)
@@ -29,8 +30,15 @@ module Textus
       (@hooks[event] ||= []) << { name: name.to_sym, callable: blk }
     end
 
-    def fetcher(name)
-      @fetchers[name.to_sym] or raise UsageError.new("unknown fetcher: #{name}")
+    def register_doctor_check(name, &blk)
+      name = name.to_sym
+      raise UsageError.new("doctor_check '#{name}' already registered") if @doctor_checks.key?(name)
+
+      @doctor_checks[name] = blk
+    end
+
+    def action(name)
+      @actions[name.to_sym] or raise UsageError.new("unknown action: #{name}")
     end
 
     def reducer(name)
@@ -41,8 +49,13 @@ module Textus
       @hooks[event.to_sym] || []
     end
 
-    def fetcher_names = @fetchers.keys
-    def reducer_names = @reducers.keys
-    def hook_events   = @hooks.keys
+    def doctor_check(name)
+      @doctor_checks[name.to_sym] or raise UsageError.new("unknown doctor_check: #{name}")
+    end
+
+    def action_names        = @actions.keys
+    def reducer_names       = @reducers.keys
+    def hook_events         = @hooks.keys
+    def doctor_check_names  = @doctor_checks.keys
   end
 end

data/lib/textus/extensions.rb

@@ -15,8 +15,8 @@ module Textus
       raise UsageError.new("no active registry; extension code must be loaded by a Store")
   end
 
-  def self.fetcher(name, &)
-    current_registry.register_fetcher(name, &)
+  def self.action(name, &)
+    current_registry.register_action(name, &)
   end
 
   def self.reducer(name, &)
@@ -26,4 +26,8 @@ module Textus
   def self.hook(event, name, &)
     current_registry.register_hook(event, name, &)
   end
+
+  def self.doctor_check(name, &)
+    current_registry.register_doctor_check(name, &)
+  end
 end

data/lib/textus/init.rb

@@ -5,7 +5,7 @@ module Textus
     ZONES = %w[canon working intake pending derived].freeze
 
     DEFAULT_MANIFEST = <<~YAML
-      version: textus/1
+      version: textus/2
       zones:
         - { name: canon,   writable_by: [human] }
         - { name: working, writable_by: [human, ai, script] }
@@ -31,12 +31,13 @@ module Textus
       File.write(File.join(target_root, "extensions", "README.md"), <<~MD)
         # Extensions
 
-        Drop one Ruby file per extension. Three verbs are available:
+        Drop one Ruby file per extension. Four verbs are available:
 
         ```ruby
-        Textus.fetcher(:name)        { |config:, store:| ... }
-        Textus.reducer(:name)        { |rows:, config:| ... }
-        Textus.hook(:event, :name)   { |key:, envelope:, store:, **kw| ... }
+        Textus.action(:name)         { |config:, store:, args:| ... }
+        Textus.reducer(:name)        { |rows:, config:|         ... }
+        Textus.hook(:event, :name)   { |key:, envelope:, **kw|  ... }
+        Textus.doctor_check(:name)   { |store:|                 ... }
         ```
 
         Events: :put, :delete, :refresh, :build, :accept.

data/lib/textus/intro.rb

@@ -6,14 +6,14 @@ module Textus
   #
   # Intro is side-effect-free.
   module Intro
-    PROTOCOL_ID = "textus/1".freeze
+    PROTOCOL_ID = PROTOCOL
 
     # Conventional zone purposes. Unknown zones (declared in the manifest
     # but not listed here) get no `purpose` field.
     ZONE_PURPOSES = {
       "canon" => "slow-changing identity; human-only writes",
       "working" => "active project state; humans, AI, and scripts share this surface",
-      "intake" => "declared external inputs; script-refreshed via fetchers",
+      "intake" => "declared external inputs; script-refreshed via actions",
       "pending" => "AI proposals awaiting human accept",
       "derived" => "build-computed outputs; never hand-edited",
     }.freeze
@@ -22,7 +22,7 @@ module Textus
       "human" => "edit files in canon/working zones, then 'textus put KEY --as=human'",
       "ai" => "propose changes by writing 'pending.*' entries with --as=ai and a 'proposal:' frontmatter block; " \
               "a human runs 'textus accept' to apply",
-      "script" => "refresh intake entries with 'textus refresh KEY --as=script' (uses the entry's declared fetcher)",
+      "script" => "refresh intake entries with 'textus refresh KEY --as=script' (uses the entry's declared action)",
       "build" => "'textus build' computes derived entries from projections; derived files are never hand-edited",
     }.freeze
 
@@ -32,7 +32,7 @@ module Textus
     CLI_VERBS = [
       { "name" => "intro",    "summary" => "this output — orientation for agents and tools" },
       { "name" => "list",     "summary" => "enumerate keys (optional --prefix)" },
-      { "name" => "get",      "summary" => "read an entry; envelope with frontmatter, body, uid, etag" },
+      { "name" => "get",      "summary" => "read an entry; envelope with _meta, body, uid, etag" },
       { "name" => "where",    "summary" => "resolve a key to its zone and path without reading" },
       { "name" => "schema",   "summary" => "field shape for a key family" },
       { "name" => "put",      "summary" => "write an entry; --as=<role>, --stdin payload" },
@@ -40,11 +40,11 @@ module Textus
       { "name" => "mv",       "summary" => "rename a key in place; uid preserved, audit row written" },
       { "name" => "delete",   "summary" => "delete an entry; --as=<role>" },
       { "name" => "build",    "summary" => "materialize derived entries; publish_to and publish_each fan out copies" },
-      { "name" => "refresh",  "summary" => "run a fetcher for an intake entry" },
+      { "name" => "refresh",  "summary" => "run an action for an intake entry" },
       { "name" => "stale",    "summary" => "list derived/intake entries past their freshness check" },
       { "name" => "doctor",   "summary" => "health-check the store (missing schemas, illegal keys, sentinel drift, etc.)" },
       { "name" => "migrate-keys", "summary" => "rename files whose basenames violate the strict key grammar" },
-      { "name" => "extensions", "summary" => "list registered reducers/fetchers/hooks" },
+      { "name" => "extensions", "summary" => "list registered actions, reducers, doctor_checks, declared hooks" },
     ].freeze
 
     def self.run(store)
@@ -80,7 +80,7 @@ module Textus
           "owner" => e.owner,
           "format" => e.format,
           "derived" => derived,
-          "intake" => !e.fetcher.nil?,
+          "intake" => !e.action.nil?,
           "publish_to" => Array(e.publish_to),
           "publish_each" => e.publish_each,
         }
@@ -90,13 +90,15 @@ module Textus
     def self.extensions_for(store)
       reg = store.registry
       reducers = reg.reducer_names.map(&:to_s).sort
-      fetchers = reg.fetcher_names.map(&:to_s).sort
+      actions = reg.action_names.map(&:to_s).sort
+      doctor_checks = reg.doctor_check_names.map(&:to_s).sort
       hooks = reg.hook_events.flat_map do |evt|
         reg.hooks(evt).map { |h| { "event" => evt.to_s, "name" => h[:name].to_s } }
       end.sort_by { |h| [h["event"], h["name"]] }
       {
         "reducers" => reducers,
-        "fetchers" => fetchers,
+        "actions" => actions,
+        "doctor_checks" => doctor_checks,
         "hooks" => hooks,
       }
     end