textus 0.4.0 → 0.8.0

data/lib/textus/refresh.rb

@@ -2,32 +2,32 @@ require "timeout"
 
 module Textus
   module Refresh
-    ACTION_TIMEOUT_SECONDS = 2
+    FETCH_TIMEOUT_SECONDS = 2
 
     def self.call(store, key, as:)
       mentry, path, = store.manifest.resolve(key)
-      raise UsageError.new("no action declared for '#{key}'") unless mentry.action
+      raise UsageError.new("no fetch declared for '#{key}'") unless mentry.fetch
 
       before_etag = File.exist?(path) ? Etag.for_file(path) : nil
-      callable = store.registry.action(mentry.action)
-      view = StoreView.new(store, writable: true, as: as)
+      callable = store.registry.rpc_callable(:fetch, mentry.fetch)
+      view = Store::View.new(store, writable: true, as: as)
       result =
         begin
-          Timeout.timeout(ACTION_TIMEOUT_SECONDS) do
-            callable.call(config: mentry.action_config, store: view, args: {})
+          Timeout.timeout(FETCH_TIMEOUT_SECONDS) do
+            callable.call(store: view, config: mentry.fetch_config, args: {})
           end
         rescue Timeout::Error
-          raise UsageError.new("action '#{mentry.action}' exceeded #{ACTION_TIMEOUT_SECONDS}s timeout")
+          raise UsageError.new("fetch '#{mentry.fetch}' exceeded #{FETCH_TIMEOUT_SECONDS}s timeout")
         rescue Textus::Error
           raise
         rescue StandardError => e
-          raise UsageError.new("action '#{mentry.action}' raised: #{e.class}: #{e.message}")
+          raise UsageError.new("fetch '#{mentry.fetch}' raised: #{e.class}: #{e.message}")
         end
 
       normalized = normalize_action_result(result, format: mentry.format)
       envelope = store.put(
         key,
-        frontmatter: normalized[:frontmatter],
+        meta: normalized[:meta],
         body: normalized[:body],
         content: normalized[:content],
         as: as,
@@ -45,28 +45,28 @@ module Textus
       envelope
     end
 
-    # Normalize the three accepted action return shapes into the store's
+    # Normalize the three accepted fetch return shapes into the store's
     # internal {frontmatter, body, content} representation.
     def self.normalize_action_result(res, format:)
       res = res.transform_keys(&:to_s) if res.is_a?(Hash)
       res ||= {}
-      fm      = res["frontmatter"]
+      # Accept both legacy :frontmatter/:_meta key names from fetch hooks.
+      meta_val = res["_meta"] || res["frontmatter"]
       body    = res["body"]
       content = res["content"]
 
       case format
       when "markdown"
-        { frontmatter: fm || {}, body: body.to_s, content: nil }
+        { meta: meta_val || {}, body: body.to_s, content: nil }
       when "text"
-        { frontmatter: {}, body: body.to_s, content: nil }
+        { meta: {}, body: body.to_s, content: nil }
       when "json", "yaml"
         if !content.nil?
-          meta = content.is_a?(Hash) && content["_meta"].is_a?(Hash) ? content["_meta"] : {}
-          { frontmatter: meta, body: nil, content: content }
+          { meta: meta_val || {}, body: nil, content: content }
         elsif !body.nil?
-          { frontmatter: {}, body: body.to_s, content: nil }
+          { meta: {}, body: body.to_s, content: nil }
         else
-          raise UsageError.new("action for #{format} returned neither content nor body")
+          raise UsageError.new("fetch for #{format} returned neither content nor body")
         end
       else
         raise UsageError.new("unknown format #{format.inspect}")

data/lib/textus/schema/tools.rb

@@ -0,0 +1,89 @@
+require "yaml"
+require "fileutils"
+
+module Textus
+  class Schema
+    module Tools
+      # textus schema init NAME --from=KEY  → infer YAML schema from an entry's frontmatter
+      def self.init(store, name:, from:)
+        env = store.get(from)
+        meta = env["_meta"]
+        schema = {
+          "name" => name,
+          "required" => meta.keys,
+          "optional" => [],
+          "fields" => meta.each_with_object({}) { |(k, v), h| h[k] = { "type" => infer_type(v) } },
+        }
+        FileUtils.mkdir_p(File.join(store.root, "schemas"))
+        target = File.join(store.root, "schemas", "#{name}.yaml")
+        File.write(target, YAML.dump(schema))
+        { "protocol" => PROTOCOL, "schema_name" => name, "path" => target }
+      end
+
+      # textus schema diff NAME  → list keys whose frontmatter violates the schema
+      def self.diff(store, name:)
+        schema = load_schema(store, name)
+        drift = []
+        store.manifest.enumerate.each do |row|
+          env = store.get(row[:key])
+          begin
+            schema.validate!(env["_meta"])
+          rescue SchemaViolation => e
+            drift << { "key" => row[:key], "details" => e.details }
+          end
+        end
+        { "protocol" => PROTOCOL, "schema_name" => name, "drift" => drift }
+      end
+
+      # textus schema migrate NAME --rename=OLD:NEW → rewrites frontmatter across affected entries
+      # If --rename is omitted, falls back to schema.evolution.migrate_from.
+      def self.migrate(store, name:, rename: nil)
+        renames =
+          if rename
+            old_field, new_field = rename.split(":", 2)
+            raise UsageError.new("--rename=OLD:NEW") unless old_field && new_field && !new_field.empty?
+
+            { old_field => new_field }
+          else
+            load_schema(store, name).evolution["migrate_from"] || {}
+          end
+        raise UsageError.new("schema migrate needs --rename=OLD:NEW or schema.evolution.migrate_from") if renames.empty?
+
+        touched = []
+        store.manifest.enumerate.each do |row|
+          env = store.get(row[:key])
+          meta = env["_meta"]
+          changed = false
+          renames.each do |old, new|
+            if meta.key?(old)
+              meta[new] = meta.delete(old)
+              changed = true
+            end
+          end
+          next unless changed
+
+          store.put(row[:key], meta: meta, body: env["body"], as: "human")
+          touched << row[:key]
+        end
+        { "protocol" => PROTOCOL, "migrated" => touched, "renames" => renames }
+      end
+
+      def self.infer_type(value)
+        case value
+        when String  then "string"
+        when Numeric then "number"
+        when true, false then "boolean"
+        when Array   then "array"
+        when Hash    then "object"
+        else "string"
+        end
+      end
+
+      def self.load_schema(store, name)
+        store.schema_for(name)
+      rescue IoError
+        raise UsageError.new("schema not found: #{name}")
+      end
+    end
+  end
+end

data/lib/textus/store/audit_log.rb

@@ -0,0 +1,71 @@
+require "json"
+require "time"
+
+module Textus
+  class Store
+    class AuditLog
+      def initialize(root)
+        @path = File.join(root, "audit.log")
+      end
+
+      def last_writer_for(key)
+        return nil unless File.exist?(@path)
+
+        last_role = nil
+        File.foreach(@path) do |line|
+          parsed = parse_row(line.chomp)
+          next unless parsed
+          next unless parsed["key"] == key
+          next unless %w[put delete].include?(parsed["verb"])
+
+          last_role = parsed["role"]
+        end
+        last_role
+      end
+
+      def append(role:, verb:, key:, etag_before:, etag_after:, extras: nil)
+        row = {
+          "ts" => Time.now.utc.iso8601,
+          "role" => role,
+          "verb" => verb,
+          "key" => key,
+          "etag_before" => etag_before,
+          "etag_after" => etag_after,
+        }
+
+        if extras.is_a?(Hash) && !extras.empty?
+          extras = extras.dup
+          %w[from_key to_key uid].each do |k|
+            row[k] = extras.delete(k) if extras.key?(k)
+          end
+          row["extras"] = extras unless extras.empty?
+        end
+
+        File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o644) do |f|
+          f.flock(File::LOCK_EX)
+          f.write(JSON.generate(row) + "\n")
+        end
+      end
+
+      private
+
+      def parse_row(line)
+        return nil if line.empty?
+
+        if line.start_with?("{")
+          JSON.parse(line)
+        else
+          # Legacy TSV (pre-0.5): read-only support retained for on-disk logs
+          # written by older textus versions. Never written by current code.
+          # Format: ts, role, verb, key, etag_before, etag_after [, json_extras]
+          fields = line.split("\t")
+          return nil if fields.length < 4
+
+          { "ts" => fields[0], "role" => fields[1], "verb" => fields[2], "key" => fields[3] }
+        end
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/textus/store/mover.rb

@@ -0,0 +1,121 @@
+require "fileutils"
+
+module Textus
+  class Store
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+    class Mover
+      def initialize(reader:, writer:, manifest:, audit_log:)
+        @reader = reader
+        @writer = writer
+        @manifest = manifest
+        @audit_log = audit_log
+      end
+
+      def call(old_key, new_key, as: Role::DEFAULT, dry_run: false)
+        @manifest.validate_key!(old_key)
+        @manifest.validate_key!(new_key)
+        raise UsageError.new("mv: old and new keys are identical") if old_key == new_key
+
+        old_mentry, old_path, = @manifest.resolve(old_key)
+        raise UnknownKey.new(old_key) unless File.exist?(old_path)
+
+        new_mentry, new_path, = @manifest.resolve(new_key)
+
+        if old_mentry.zone != new_mentry.zone
+          raise UsageError.new(
+            "mv: cross-zone move refused (#{old_mentry.zone} → #{new_mentry.zone}). " \
+            "Use put+delete for cross-zone moves.",
+          )
+        end
+        if old_mentry.format != new_mentry.format
+          raise UsageError.new(
+            "mv: format mismatch (#{old_mentry.format} → #{new_mentry.format}); refusing.",
+          )
+        end
+
+        writers = @manifest.zone_writers(old_mentry.zone)
+        raise WriteForbidden.new(old_key, old_mentry.zone, writers: writers) unless writers.include?(as)
+
+        raise UsageError.new("mv: target '#{new_key}' already exists at #{new_path}") if File.exist?(new_path)
+
+        # Mint uid before the move so the audit row carries it.
+        pre_env = @reader.get(old_key)
+        current_uid = pre_env["uid"]
+        etag_before = pre_env["etag"]
+
+        if dry_run
+          return {
+            "protocol" => PROTOCOL, "ok" => true, "dry_run" => true,
+            "from_key" => old_key, "to_key" => new_key,
+            "from_path" => old_path, "to_path" => new_path,
+            "uid" => current_uid
+          }
+        end
+
+        if current_uid.nil?
+          # Write the uid in place first so the source file carries it before mv.
+          pre_env = @writer.put(old_key,
+                                meta: pre_env["_meta"],
+                                body: pre_env["body"],
+                                content: pre_env["content"],
+                                as: as,
+                                suppress_events: true)
+          current_uid = pre_env["uid"]
+          etag_before = pre_env["etag"]
+        end
+
+        FileUtils.mkdir_p(File.dirname(new_path))
+        FileUtils.mv(old_path, new_path)
+        rewrite_name_for_mv!(new_mentry, new_path, new_key)
+        etag_after = Etag.for_file(new_path)
+
+        @audit_log.append(
+          role: as, verb: "mv", key: new_key,
+          etag_before: etag_before, etag_after: etag_after,
+          extras: {
+            "from_key" => old_key, "to_key" => new_key,
+            "from_path" => old_path, "to_path" => new_path,
+            "uid" => current_uid
+          }
+        )
+
+        env = @reader.get(new_key)
+        {
+          "protocol" => PROTOCOL, "ok" => true,
+          "from_key" => old_key, "to_key" => new_key,
+          "from_path" => old_path, "to_path" => new_path,
+          "uid" => current_uid,
+          "envelope" => env
+        }
+      end
+
+      private
+
+      # If the moved file carries a `name:` field (markdown) or `_meta.name`
+      # (json/yaml), rewrite it to the new basename so enforce_name_match! stays
+      # happy on the next read. Only touches the bytes when name actually changes.
+      def rewrite_name_for_mv!(mentry, new_path, new_key)
+        strategy = Entry.for_format(mentry.format)
+        raw = File.binread(new_path)
+        parsed = strategy.parse(raw, path: new_path)
+        basename = new_key.split(".").last
+
+        case mentry.format
+        when "markdown"
+          meta = parsed["_meta"] || {}
+          return unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
+
+          meta = meta.merge("name" => basename)
+          File.binwrite(new_path, strategy.serialize(meta: meta, body: parsed["body"]))
+        when "json", "yaml"
+          meta = parsed["_meta"]
+          return unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
+
+          new_meta = meta.merge("name" => basename)
+          File.binwrite(new_path, strategy.serialize(meta: new_meta, body: "", content: parsed["content"]))
+        end
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+  end
+end

data/lib/textus/store/reader.rb

@@ -0,0 +1,67 @@
+module Textus
+  class Store
+    class Reader
+      def initialize(store)
+        @store = store
+        @manifest = store.manifest
+      end
+
+      def get(key)
+        mentry, path, = @manifest.resolve(key)
+        raise UnknownKey.new(key, suggestions: @manifest.suggestions_for(key)) unless File.exist?(path)
+
+        raw = File.binread(path)
+        parsed = Entry.for_format(mentry.format).parse(raw, path: path)
+        meta = parsed["_meta"]
+        content = parsed["content"]
+        @store.writer.enforce_name_match!(path, meta, mentry.format)
+        schema = @store.schema_for(mentry.schema)
+        Entry.for_format(mentry.format).validate_against(schema, parsed) if schema
+        Envelope.build(
+          key: key, mentry: mentry, path: path,
+          meta: meta, body: parsed["body"],
+          etag: Etag.for_bytes(raw), content: content
+        )
+      end
+
+      def list(prefix: nil, zone: nil)
+        rows = @manifest.enumerate(prefix: prefix)
+        rows = rows.select { |r| r[:manifest_entry].zone == zone } if zone
+        rows.map { |row| { "key" => row[:key], "zone" => row[:manifest_entry].zone, "path" => row[:path] } }
+      end
+
+      def where(key)
+        mentry, path, = @manifest.resolve(key)
+        { "protocol" => PROTOCOL, "key" => key, "zone" => mentry.zone, "owner" => mentry.owner, "path" => path }
+      end
+
+      def schema_envelope(key)
+        mentry, = @manifest.resolve(key)
+        schema = @store.schema_for(mentry.schema)
+        { "protocol" => PROTOCOL, "key" => key, "schema_ref" => mentry.schema, "schema" => schema&.to_h }
+      end
+
+      # Returns the Textus UID for a key (or nil if the entry has none yet).
+      # Raises UnknownKey if the key doesn't resolve to a real file.
+      def uid(key)
+        get(key)["uid"]
+      end
+
+      def deps(key)   = Dependencies.deps_of(@manifest, key)
+      def rdeps(key)  = Dependencies.rdeps_of(@manifest, key)
+      def published   = Dependencies.published_of(@manifest)
+
+      def stale(prefix: nil, zone: nil)
+        Staleness.new(manifest: @manifest).call(prefix: prefix, zone: zone)
+      end
+
+      def validate_all
+        Validator.new(
+          reader: self, manifest: @manifest,
+          audit_log: @store.audit_log,
+          schema_for: ->(name) { @store.schema_for(name) }
+        ).call
+      end
+    end
+  end
+end

data/lib/textus/store/staleness.rb

@@ -0,0 +1,133 @@
+require "time"
+
+module Textus
+  class Store
+    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/BlockLength
+    class Staleness
+      def initialize(manifest:)
+        @manifest = manifest
+      end
+
+      def call(prefix: nil, zone: nil)
+        out = []
+        @manifest.entries.each do |mentry|
+          next unless mentry.zone == "derived"
+          next if zone && mentry.zone != zone
+
+          gen = mentry.generator
+          next unless gen
+          next if prefix && !(mentry.key == prefix || mentry.key.start_with?("#{prefix}."))
+
+          path = Textus::Key::Path.resolve(@manifest, mentry)
+
+          unless File.exist?(path)
+            out << stale_row(mentry, path, "derived entry has never been generated")
+            next
+          end
+
+          raw = File.binread(path)
+          parsed = Entry.for_format(mentry.format).parse(raw, path: path)
+          generated_at = parsed["_meta"].dig("generated", "at")
+          unless generated_at
+            out << stale_row(mentry, path, "missing generated.at frontmatter")
+            next
+          end
+          gen_time = begin
+            Time.parse(generated_at.to_s)
+          rescue StandardError
+            nil
+          end
+          unless gen_time
+            out << stale_row(mentry, path, "unparseable generated.at: #{generated_at.inspect}")
+            next
+          end
+
+          offender = newest_source_after(gen, gen_time)
+          out << stale_row(mentry, path, "source '#{offender}' modified after generated.at") if offender
+        end
+
+        @manifest.entries.each do |mentry|
+          next unless mentry.fetch
+          next if zone && mentry.zone != zone
+          next if prefix && !(mentry.key == prefix || mentry.key.start_with?("#{prefix}."))
+
+          ttl = parse_ttl(mentry.ttl)
+          next unless ttl
+
+          path = Textus::Key::Path.resolve(@manifest, mentry)
+
+          unless File.exist?(path)
+            out << intake_stale_row(mentry, path, "never refreshed")
+            next
+          end
+
+          meta = Entry.for_format(mentry.format).parse(File.binread(path), path: path)["_meta"]
+          last_str = meta["last_refreshed_at"]
+          if last_str.nil?
+            out << intake_stale_row(mentry, path, "never refreshed (no last_refreshed_at)")
+            next
+          end
+
+          last = begin
+            Time.parse(last_str.to_s)
+          rescue StandardError
+            nil
+          end
+          out << intake_stale_row(mentry, path, "ttl exceeded (#{ttl}s)") if last.nil? || (Time.now - last) > ttl
+        end
+
+        out
+      end
+
+      private
+
+      def newest_source_after(gen, gen_time)
+        Array(gen["sources"]).each do |src|
+          if src.match?(/\A[a-z0-9.][a-z0-9._-]*\z/) && !src.include?("/")
+            @manifest.enumerate(prefix: src).each do |row|
+              return src if File.mtime(row[:path]) > gen_time
+            end
+          else
+            abs = File.absolute_path?(src) ? src : File.join(File.dirname(@manifest.root), src)
+            if File.directory?(abs)
+              Dir.glob(File.join(abs, "**", "*")).each do |fp|
+                next unless File.file?(fp)
+                return src if File.mtime(fp) > gen_time
+              end
+            elsif File.exist?(abs)
+              return src if File.mtime(abs) > gen_time
+            end
+          end
+        end
+        nil
+      end
+
+      def parse_ttl(s)
+        return nil unless s
+
+        m = s.to_s.match(/\A(\d+)([smhd])\z/) or return nil
+        n = m[1].to_i
+        case m[2]
+        when "s" then n
+        when "m" then n * 60
+        when "h" then n * 3600
+        when "d" then n * 86_400
+        end
+      end
+
+      def intake_stale_row(mentry, path, reason)
+        { "key" => mentry.key, "path" => path, "fetch" => mentry.fetch, "reason" => reason }
+      end
+
+      def stale_row(mentry, path, reason)
+        {
+          "key" => mentry.key,
+          "path" => path,
+          "generator" => mentry.generator,
+          "reason" => reason,
+        }
+      end
+    end
+    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/BlockLength
+  end
+end

data/lib/textus/store/validator.rb

@@ -0,0 +1,56 @@
+module Textus
+  class Store
+    class Validator
+      def initialize(reader:, manifest:, audit_log:, schema_for:)
+        @reader = reader
+        @manifest = manifest
+        @audit_log = audit_log
+        @schema_for = schema_for
+      end
+
+      def call
+        violations = []
+        @manifest.enumerate.each do |row|
+          begin
+            @reader.get(row[:key])
+          rescue Textus::Error => e
+            violations << { "key" => row[:key], "code" => e.code, "message" => e.message }
+          end
+        end
+
+        @manifest.enumerate.each do |row|
+          mentry = row[:manifest_entry]
+          next unless mentry.schema
+
+          schema = @schema_for.call(mentry.schema)
+          next unless schema
+
+          env = begin
+            @reader.get(row[:key])
+          rescue StandardError
+            next
+          end
+          last_writer = @audit_log.last_writer_for(row[:key])
+          next if last_writer.nil?
+
+          env["_meta"].each_key do |field|
+            owner = schema.maintained_by(field)
+            next if owner.nil?
+            next if last_writer == owner
+            next if last_writer == "human"
+
+            violations << {
+              "key" => row[:key],
+              "code" => "role_authority",
+              "field" => field,
+              "expected" => owner,
+              "last_writer" => last_writer,
+            }
+          end
+        end
+
+        { "protocol" => PROTOCOL, "ok" => violations.empty?, "violations" => violations }
+      end
+    end
+  end
+end

data/lib/textus/store/view.rb

@@ -0,0 +1,29 @@
+module Textus
+  class Store
+    class View
+      READ_METHODS  = %i[get list where schema_envelope deps rdeps published stale validate_all].freeze
+      WRITE_METHODS = %i[put delete accept].freeze
+
+      def initialize(store, writable: false, as: nil)
+        raise UsageError.new("writable Store::View requires an as: role") if writable && (as.nil? || as.to_s.empty?)
+
+        @store = store
+        @writable = writable
+        @as = as
+      end
+
+      READ_METHODS.each do |m|
+        define_method(m) { |*args, **kw| @store.reader.public_send(m, *args, **kw) }
+      end
+
+      WRITE_METHODS.each do |m|
+        define_method(m) do |*args, **kw|
+          raise UsageError.new("Store::View is read-only") unless @writable
+
+          kw[:as] = @as unless kw.key?(:as)
+          @store.public_send(m, *args, **kw)
+        end
+      end
+    end
+  end
+end