textus 0.5.0 → 0.8.0

data/lib/textus/refresh.rb

@@ -2,26 +2,26 @@ require "timeout"
 
 module Textus
   module Refresh
-    ACTION_TIMEOUT_SECONDS = 2
+    FETCH_TIMEOUT_SECONDS = 2
 
     def self.call(store, key, as:)
       mentry, path, = store.manifest.resolve(key)
-      raise UsageError.new("no action declared for '#{key}'") unless mentry.action
+      raise UsageError.new("no fetch declared for '#{key}'") unless mentry.fetch
 
       before_etag = File.exist?(path) ? Etag.for_file(path) : nil
-      callable = store.registry.action(mentry.action)
-      view = StoreView.new(store, writable: true, as: as)
+      callable = store.registry.rpc_callable(:fetch, mentry.fetch)
+      view = Store::View.new(store, writable: true, as: as)
       result =
         begin
-          Timeout.timeout(ACTION_TIMEOUT_SECONDS) do
-            callable.call(config: mentry.action_config, store: view, args: {})
+          Timeout.timeout(FETCH_TIMEOUT_SECONDS) do
+            callable.call(store: view, config: mentry.fetch_config, args: {})
           end
         rescue Timeout::Error
-          raise UsageError.new("action '#{mentry.action}' exceeded #{ACTION_TIMEOUT_SECONDS}s timeout")
+          raise UsageError.new("fetch '#{mentry.fetch}' exceeded #{FETCH_TIMEOUT_SECONDS}s timeout")
         rescue Textus::Error
           raise
         rescue StandardError => e
-          raise UsageError.new("action '#{mentry.action}' raised: #{e.class}: #{e.message}")
+          raise UsageError.new("fetch '#{mentry.fetch}' raised: #{e.class}: #{e.message}")
         end
 
       normalized = normalize_action_result(result, format: mentry.format)
@@ -45,12 +45,12 @@ module Textus
       envelope
     end
 
-    # Normalize the three accepted action return shapes into the store's
+    # Normalize the three accepted fetch return shapes into the store's
     # internal {frontmatter, body, content} representation.
     def self.normalize_action_result(res, format:)
       res = res.transform_keys(&:to_s) if res.is_a?(Hash)
       res ||= {}
-      # Accept both legacy :frontmatter/:_meta key names from actions.
+      # Accept both legacy :frontmatter/:_meta key names from fetch hooks.
       meta_val = res["_meta"] || res["frontmatter"]
       body    = res["body"]
       content = res["content"]
@@ -66,7 +66,7 @@ module Textus
         elsif !body.nil?
           { meta: {}, body: body.to_s, content: nil }
         else
-          raise UsageError.new("action for #{format} returned neither content nor body")
+          raise UsageError.new("fetch for #{format} returned neither content nor body")
         end
       else
         raise UsageError.new("unknown format #{format.inspect}")

data/lib/textus/schema/tools.rb

@@ -0,0 +1,89 @@
+require "yaml"
+require "fileutils"
+
+module Textus
+  class Schema
+    module Tools
+      # textus schema init NAME --from=KEY  → infer YAML schema from an entry's frontmatter
+      def self.init(store, name:, from:)
+        env = store.get(from)
+        meta = env["_meta"]
+        schema = {
+          "name" => name,
+          "required" => meta.keys,
+          "optional" => [],
+          "fields" => meta.each_with_object({}) { |(k, v), h| h[k] = { "type" => infer_type(v) } },
+        }
+        FileUtils.mkdir_p(File.join(store.root, "schemas"))
+        target = File.join(store.root, "schemas", "#{name}.yaml")
+        File.write(target, YAML.dump(schema))
+        { "protocol" => PROTOCOL, "schema_name" => name, "path" => target }
+      end
+
+      # textus schema diff NAME  → list keys whose frontmatter violates the schema
+      def self.diff(store, name:)
+        schema = load_schema(store, name)
+        drift = []
+        store.manifest.enumerate.each do |row|
+          env = store.get(row[:key])
+          begin
+            schema.validate!(env["_meta"])
+          rescue SchemaViolation => e
+            drift << { "key" => row[:key], "details" => e.details }
+          end
+        end
+        { "protocol" => PROTOCOL, "schema_name" => name, "drift" => drift }
+      end
+
+      # textus schema migrate NAME --rename=OLD:NEW → rewrites frontmatter across affected entries
+      # If --rename is omitted, falls back to schema.evolution.migrate_from.
+      def self.migrate(store, name:, rename: nil)
+        renames =
+          if rename
+            old_field, new_field = rename.split(":", 2)
+            raise UsageError.new("--rename=OLD:NEW") unless old_field && new_field && !new_field.empty?
+
+            { old_field => new_field }
+          else
+            load_schema(store, name).evolution["migrate_from"] || {}
+          end
+        raise UsageError.new("schema migrate needs --rename=OLD:NEW or schema.evolution.migrate_from") if renames.empty?
+
+        touched = []
+        store.manifest.enumerate.each do |row|
+          env = store.get(row[:key])
+          meta = env["_meta"]
+          changed = false
+          renames.each do |old, new|
+            if meta.key?(old)
+              meta[new] = meta.delete(old)
+              changed = true
+            end
+          end
+          next unless changed
+
+          store.put(row[:key], meta: meta, body: env["body"], as: "human")
+          touched << row[:key]
+        end
+        { "protocol" => PROTOCOL, "migrated" => touched, "renames" => renames }
+      end
+
+      def self.infer_type(value)
+        case value
+        when String  then "string"
+        when Numeric then "number"
+        when true, false then "boolean"
+        when Array   then "array"
+        when Hash    then "object"
+        else "string"
+        end
+      end
+
+      def self.load_schema(store, name)
+        store.schema_for(name)
+      rescue IoError
+        raise UsageError.new("schema not found: #{name}")
+      end
+    end
+  end
+end

data/lib/textus/store/audit_log.rb

@@ -0,0 +1,71 @@
+require "json"
+require "time"
+
+module Textus
+  class Store
+    class AuditLog
+      def initialize(root)
+        @path = File.join(root, "audit.log")
+      end
+
+      def last_writer_for(key)
+        return nil unless File.exist?(@path)
+
+        last_role = nil
+        File.foreach(@path) do |line|
+          parsed = parse_row(line.chomp)
+          next unless parsed
+          next unless parsed["key"] == key
+          next unless %w[put delete].include?(parsed["verb"])
+
+          last_role = parsed["role"]
+        end
+        last_role
+      end
+
+      def append(role:, verb:, key:, etag_before:, etag_after:, extras: nil)
+        row = {
+          "ts" => Time.now.utc.iso8601,
+          "role" => role,
+          "verb" => verb,
+          "key" => key,
+          "etag_before" => etag_before,
+          "etag_after" => etag_after,
+        }
+
+        if extras.is_a?(Hash) && !extras.empty?
+          extras = extras.dup
+          %w[from_key to_key uid].each do |k|
+            row[k] = extras.delete(k) if extras.key?(k)
+          end
+          row["extras"] = extras unless extras.empty?
+        end
+
+        File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o644) do |f|
+          f.flock(File::LOCK_EX)
+          f.write(JSON.generate(row) + "\n")
+        end
+      end
+
+      private
+
+      def parse_row(line)
+        return nil if line.empty?
+
+        if line.start_with?("{")
+          JSON.parse(line)
+        else
+          # Legacy TSV (pre-0.5): read-only support retained for on-disk logs
+          # written by older textus versions. Never written by current code.
+          # Format: ts, role, verb, key, etag_before, etag_after [, json_extras]
+          fields = line.split("\t")
+          return nil if fields.length < 4
+
+          { "ts" => fields[0], "role" => fields[1], "verb" => fields[2], "key" => fields[3] }
+        end
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/textus/store/mover.rb

@@ -4,19 +4,22 @@ module Textus
   class Store
     # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
     class Mover
-      def initialize(store)
-        @store = store
+      def initialize(reader:, writer:, manifest:, audit_log:)
+        @reader = reader
+        @writer = writer
+        @manifest = manifest
+        @audit_log = audit_log
       end
 
       def call(old_key, new_key, as: Role::DEFAULT, dry_run: false)
-        @store.manifest.validate_key!(old_key)
-        @store.manifest.validate_key!(new_key)
+        @manifest.validate_key!(old_key)
+        @manifest.validate_key!(new_key)
         raise UsageError.new("mv: old and new keys are identical") if old_key == new_key
 
-        old_mentry, old_path, = @store.manifest.resolve(old_key)
+        old_mentry, old_path, = @manifest.resolve(old_key)
         raise UnknownKey.new(old_key) unless File.exist?(old_path)
 
-        new_mentry, new_path, = @store.manifest.resolve(new_key)
+        new_mentry, new_path, = @manifest.resolve(new_key)
 
         if old_mentry.zone != new_mentry.zone
           raise UsageError.new(
@@ -30,13 +33,13 @@ module Textus
           )
         end
 
-        writers = @store.manifest.zone_writers(old_mentry.zone)
+        writers = @manifest.zone_writers(old_mentry.zone)
         raise WriteForbidden.new(old_key, old_mentry.zone, writers: writers) unless writers.include?(as)
 
         raise UsageError.new("mv: target '#{new_key}' already exists at #{new_path}") if File.exist?(new_path)
 
         # Mint uid before the move so the audit row carries it.
-        pre_env = @store.get(old_key)
+        pre_env = @reader.get(old_key)
         current_uid = pre_env["uid"]
         etag_before = pre_env["etag"]
 
@@ -51,12 +54,12 @@ module Textus
 
         if current_uid.nil?
           # Write the uid in place first so the source file carries it before mv.
-          pre_env = @store.put(old_key,
-                               meta: pre_env["_meta"],
-                               body: pre_env["body"],
-                               content: pre_env["content"],
-                               as: as,
-                               suppress_events: true)
+          pre_env = @writer.put(old_key,
+                                meta: pre_env["_meta"],
+                                body: pre_env["body"],
+                                content: pre_env["content"],
+                                as: as,
+                                suppress_events: true)
           current_uid = pre_env["uid"]
           etag_before = pre_env["etag"]
         end
@@ -66,7 +69,7 @@ module Textus
         rewrite_name_for_mv!(new_mentry, new_path, new_key)
         etag_after = Etag.for_file(new_path)
 
-        @store.audit_log.append(
+        @audit_log.append(
           role: as, verb: "mv", key: new_key,
           etag_before: etag_before, etag_after: etag_after,
           extras: {
@@ -76,7 +79,7 @@ module Textus
           }
         )
 
-        env = @store.get(new_key)
+        env = @reader.get(new_key)
         {
           "protocol" => PROTOCOL, "ok" => true,
           "from_key" => old_key, "to_key" => new_key,

data/lib/textus/store/reader.rb

@@ -0,0 +1,67 @@
+module Textus
+  class Store
+    class Reader
+      def initialize(store)
+        @store = store
+        @manifest = store.manifest
+      end
+
+      def get(key)
+        mentry, path, = @manifest.resolve(key)
+        raise UnknownKey.new(key, suggestions: @manifest.suggestions_for(key)) unless File.exist?(path)
+
+        raw = File.binread(path)
+        parsed = Entry.for_format(mentry.format).parse(raw, path: path)
+        meta = parsed["_meta"]
+        content = parsed["content"]
+        @store.writer.enforce_name_match!(path, meta, mentry.format)
+        schema = @store.schema_for(mentry.schema)
+        Entry.for_format(mentry.format).validate_against(schema, parsed) if schema
+        Envelope.build(
+          key: key, mentry: mentry, path: path,
+          meta: meta, body: parsed["body"],
+          etag: Etag.for_bytes(raw), content: content
+        )
+      end
+
+      def list(prefix: nil, zone: nil)
+        rows = @manifest.enumerate(prefix: prefix)
+        rows = rows.select { |r| r[:manifest_entry].zone == zone } if zone
+        rows.map { |row| { "key" => row[:key], "zone" => row[:manifest_entry].zone, "path" => row[:path] } }
+      end
+
+      def where(key)
+        mentry, path, = @manifest.resolve(key)
+        { "protocol" => PROTOCOL, "key" => key, "zone" => mentry.zone, "owner" => mentry.owner, "path" => path }
+      end
+
+      def schema_envelope(key)
+        mentry, = @manifest.resolve(key)
+        schema = @store.schema_for(mentry.schema)
+        { "protocol" => PROTOCOL, "key" => key, "schema_ref" => mentry.schema, "schema" => schema&.to_h }
+      end
+
+      # Returns the Textus UID for a key (or nil if the entry has none yet).
+      # Raises UnknownKey if the key doesn't resolve to a real file.
+      def uid(key)
+        get(key)["uid"]
+      end
+
+      def deps(key)   = Dependencies.deps_of(@manifest, key)
+      def rdeps(key)  = Dependencies.rdeps_of(@manifest, key)
+      def published   = Dependencies.published_of(@manifest)
+
+      def stale(prefix: nil, zone: nil)
+        Staleness.new(manifest: @manifest).call(prefix: prefix, zone: zone)
+      end
+
+      def validate_all
+        Validator.new(
+          reader: self, manifest: @manifest,
+          audit_log: @store.audit_log,
+          schema_for: ->(name) { @store.schema_for(name) }
+        ).call
+      end
+    end
+  end
+end

data/lib/textus/store/staleness.rb

@@ -4,13 +4,13 @@ module Textus
   class Store
     # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/BlockLength
     class Staleness
-      def initialize(store)
-        @store = store
+      def initialize(manifest:)
+        @manifest = manifest
       end
 
       def call(prefix: nil, zone: nil)
         out = []
-        @store.manifest.entries.each do |mentry|
+        @manifest.entries.each do |mentry|
           next unless mentry.zone == "derived"
           next if zone && mentry.zone != zone
 
@@ -18,7 +18,7 @@ module Textus
           next unless gen
           next if prefix && !(mentry.key == prefix || mentry.key.start_with?("#{prefix}."))
 
-          path = path_for_entry(mentry)
+          path = Textus::Key::Path.resolve(@manifest, mentry)
 
           unless File.exist?(path)
             out << stale_row(mentry, path, "derived entry has never been generated")
@@ -46,15 +46,15 @@ module Textus
           out << stale_row(mentry, path, "source '#{offender}' modified after generated.at") if offender
         end
 
-        @store.manifest.entries.each do |mentry|
-          next unless mentry.action
+        @manifest.entries.each do |mentry|
+          next unless mentry.fetch
           next if zone && mentry.zone != zone
           next if prefix && !(mentry.key == prefix || mentry.key.start_with?("#{prefix}."))
 
           ttl = parse_ttl(mentry.ttl)
           next unless ttl
 
-          path = path_for_entry(mentry)
+          path = Textus::Key::Path.resolve(@manifest, mentry)
 
           unless File.exist?(path)
             out << intake_stale_row(mentry, path, "never refreshed")
@@ -81,23 +81,14 @@ module Textus
 
       private
 
-      def path_for_entry(mentry)
-        primary_ext = Entry.for_format(mentry.format).extensions.first
-        if File.extname(mentry.path) == ""
-          File.join(@store.root, "zones", mentry.path + primary_ext)
-        else
-          File.join(@store.root, "zones", mentry.path)
-        end
-      end
-
       def newest_source_after(gen, gen_time)
         Array(gen["sources"]).each do |src|
           if src.match?(/\A[a-z0-9.][a-z0-9._-]*\z/) && !src.include?("/")
-            @store.manifest.enumerate(prefix: src).each do |row|
+            @manifest.enumerate(prefix: src).each do |row|
               return src if File.mtime(row[:path]) > gen_time
             end
           else
-            abs = File.absolute_path?(src) ? src : File.join(File.dirname(@store.root), src)
+            abs = File.absolute_path?(src) ? src : File.join(File.dirname(@manifest.root), src)
             if File.directory?(abs)
               Dir.glob(File.join(abs, "**", "*")).each do |fp|
                 next unless File.file?(fp)
@@ -125,7 +116,7 @@ module Textus
       end
 
       def intake_stale_row(mentry, path, reason)
-        { "key" => mentry.key, "path" => path, "action" => mentry.action, "reason" => reason }
+        { "key" => mentry.key, "path" => path, "fetch" => mentry.fetch, "reason" => reason }
       end
 
       def stale_row(mentry, path, reason)

data/lib/textus/store/validator.rb

@@ -1,33 +1,36 @@
 module Textus
   class Store
     class Validator
-      def initialize(store)
-        @store = store
+      def initialize(reader:, manifest:, audit_log:, schema_for:)
+        @reader = reader
+        @manifest = manifest
+        @audit_log = audit_log
+        @schema_for = schema_for
       end
 
       def call
         violations = []
-        @store.manifest.enumerate.each do |row|
+        @manifest.enumerate.each do |row|
           begin
-            @store.get(row[:key])
+            @reader.get(row[:key])
           rescue Textus::Error => e
             violations << { "key" => row[:key], "code" => e.code, "message" => e.message }
           end
         end
 
-        @store.manifest.enumerate.each do |row|
+        @manifest.enumerate.each do |row|
           mentry = row[:manifest_entry]
           next unless mentry.schema
 
-          schema = @store.schema_for(mentry.schema)
+          schema = @schema_for.call(mentry.schema)
           next unless schema
 
           env = begin
-            @store.get(row[:key])
+            @reader.get(row[:key])
           rescue StandardError
             next
           end
-          last_writer = @store.audit_log.last_writer_for(row[:key])
+          last_writer = @audit_log.last_writer_for(row[:key])
           next if last_writer.nil?
 
           env["_meta"].each_key do |field|

data/lib/textus/store/view.rb

@@ -0,0 +1,29 @@
+module Textus
+  class Store
+    class View
+      READ_METHODS  = %i[get list where schema_envelope deps rdeps published stale validate_all].freeze
+      WRITE_METHODS = %i[put delete accept].freeze
+
+      def initialize(store, writable: false, as: nil)
+        raise UsageError.new("writable Store::View requires an as: role") if writable && (as.nil? || as.to_s.empty?)
+
+        @store = store
+        @writable = writable
+        @as = as
+      end
+
+      READ_METHODS.each do |m|
+        define_method(m) { |*args, **kw| @store.reader.public_send(m, *args, **kw) }
+      end
+
+      WRITE_METHODS.each do |m|
+        define_method(m) do |*args, **kw|
+          raise UsageError.new("Store::View is read-only") unless @writable
+
+          kw[:as] = @as unless kw.key?(:as)
+          @store.public_send(m, *args, **kw)
+        end
+      end
+    end
+  end
+end

data/lib/textus/store/writer.rb

@@ -0,0 +1,132 @@
+require "fileutils"
+
+module Textus
+  class Store
+    # rubocop:disable Metrics/ParameterLists
+    class Writer
+      def initialize(store)
+        @store = store
+        @manifest = store.manifest
+        @reader = store.reader
+      end
+
+      def put(key, meta: nil, body: nil, content: nil, if_etag: nil, as: Role::DEFAULT, suppress_events: false)
+        @manifest.validate_key!(key)
+        mentry, path, = @manifest.resolve(key)
+        writers = @manifest.zone_writers(mentry.zone)
+        raise WriteForbidden.new(key, mentry.zone, writers: writers) unless writers.include?(as)
+
+        meta ||= {}
+        strategy = Entry.for_format(mentry.format)
+
+        existing_uid = existing_uid_for(mentry, path)
+        meta, content = ensure_uid(mentry.format, meta, content, existing_uid)
+
+        bytes, eff_meta, eff_body, eff_content = serialize_for_put(
+          mentry: mentry, path: path, strategy: strategy,
+          meta: meta, body: body, content: content
+        )
+
+        enforce_name_match!(path, eff_meta, mentry.format)
+
+        schema = @store.schema_for(mentry.schema)
+        if schema
+          Entry.for_format(mentry.format).validate_against(
+            schema,
+            { "_meta" => eff_meta, "content" => eff_content },
+          )
+        end
+
+        etag_before = File.exist?(path) ? Etag.for_file(path) : nil
+        raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && (etag_before != if_etag)
+
+        FileUtils.mkdir_p(File.dirname(path))
+        File.binwrite(path, bytes)
+        etag_after = Etag.for_bytes(bytes)
+        @store.audit_log.append(role: as, verb: "put", key: key, etag_before: etag_before, etag_after: etag_after)
+        envelope = Envelope.build(
+          key: key, mentry: mentry, path: path,
+          meta: eff_meta, body: eff_body, etag: etag_after, content: eff_content
+        )
+        @store.fire_event(:put, key: key, envelope: envelope) unless suppress_events
+        envelope
+      end
+
+      def existing_uid_for(mentry, path)
+        return nil unless File.exist?(path)
+
+        raw = File.binread(path)
+        parsed = Entry.for_format(mentry.format).parse(raw, path: path)
+        Envelope.extract_uid(parsed["_meta"])
+      rescue StandardError
+        nil
+      end
+
+      def ensure_uid(format, meta, content, existing_uid)
+        case format
+        when "markdown", "json", "yaml"
+          m = meta.is_a?(Hash) ? meta.dup : {}
+          m["uid"] = existing_uid || Store.mint_uid unless m["uid"].is_a?(String) && !m["uid"].empty?
+          [m, content]
+        else
+          [meta, content]
+        end
+      end
+
+      def enforce_name_match!(path, meta, format)
+        return unless %w[markdown json yaml].include?(format)
+        return unless meta.is_a?(Hash) && meta["name"]
+
+        ext = Entry.for_format(format).extensions.first
+        basename = File.basename(path, ext)
+        return if meta["name"] == basename
+
+        raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
+      end
+
+      def serialize_for_put(mentry:, path:, strategy:, meta:, body:, content:)
+        case mentry.format
+        when "markdown", "text"
+          bytes = strategy.serialize(meta: meta, body: body.to_s)
+          [bytes, meta, body.to_s, nil]
+        when "json", "yaml"
+          raise UsageError.new("put for #{mentry.format} requires content: or body:") if content.nil? && (body.nil? || body.to_s.empty?)
+
+          if content.nil?
+            begin
+              parsed = strategy.parse(body.to_s, path: path)
+            rescue BadFrontmatter => e
+              raise BadContent.new(path, "bad_content: #{e.message}")
+            end
+            [body.to_s, parsed["_meta"], body.to_s, parsed["content"]]
+          else
+            bytes = strategy.serialize(meta: meta, body: "", content: content)
+            [bytes, meta, bytes, content]
+          end
+        else
+          raise UsageError.new("unknown format #{mentry.format.inspect}")
+        end
+      end
+
+      def delete(key, if_etag: nil, as: Role::DEFAULT, suppress_events: false)
+        mentry, path, = @manifest.resolve(key)
+        writers = @manifest.zone_writers(mentry.zone)
+        raise WriteForbidden.new(key, mentry.zone, writers: writers) unless writers.include?(as)
+        raise UnknownKey.new(key, suggestions: @manifest.suggestions_for(key)) unless File.exist?(path)
+
+        etag_before = Etag.for_file(path)
+        raise EtagMismatch.new(key, if_etag, etag_before) if if_etag && if_etag != etag_before
+
+        File.delete(path)
+        @store.audit_log.append(role: as, verb: "delete", key: key, etag_before: etag_before, etag_after: nil)
+        @store.fire_event(:delete, key: key) unless suppress_events
+        { "protocol" => PROTOCOL, "ok" => true, "key" => key, "deleted" => true }
+      end
+
+      def accept(key, as:)
+        Proposal.accept(@store, key, as: as)
+      end
+    end
+    # rubocop:enable Metrics/ParameterLists
+  end
+end