textus 0.20.2 → 0.22.0

data/lib/textus/cli/verb/audit.rb

@@ -9,6 +9,7 @@ module Textus
         option :role_filter, "--role=ROLE"
         option :verb_filter, "--verb=V"
         option :since, "--since=ISO8601|RELATIVE"
+        option :seq_since, "--seq-since=N"
         option :correlation_id, "--correlation-id=ID"
         option :limit, "--limit=N"
 
@@ -21,6 +22,7 @@ module Textus
             role: role_filter,
             verb: verb_filter,
             since: since_time,
+            seq_since: seq_since&.to_i,
             correlation_id: correlation_id,
             limit: limit&.to_i,
           )

data/lib/textus/cli/verb/{intro.rb → boot.rb}

@@ -1,11 +1,11 @@
 module Textus
   class CLI
     class Verb
-      class Intro < Verb
-        command_name "intro"
+      class Boot < Verb
+        command_name "boot"
 
         def call(store)
-          emit(Textus::Intro.run(store))
+          emit(Textus::Boot.run(store))
         end
       end
     end

data/lib/textus/cli/verb/pulse.rb

@@ -0,0 +1,17 @@
+module Textus
+  class CLI
+    class Verb
+      class Pulse < Verb
+        command_name "pulse"
+
+        option :since, "--since=N"
+
+        def call(store)
+          ops = operations_for(store)
+          since_n = (since || "0").to_i
+          emit(ops.pulse(since: since_n))
+        end
+      end
+    end
+  end
+end

data/lib/textus/cli.rb

@@ -99,7 +99,7 @@ module Textus
           textus audit [--key=K] [--zone=Z] [--role=R] [--verb=V] [--since=X] [--correlation-id=ID] [--limit=N]
           textus blame KEY [--limit=N]
           textus doctor
-          textus intro
+          textus boot
 
           textus key {mv,uid,normalize}
           textus rule {list,explain}

data/lib/textus/errors.rb

@@ -215,4 +215,20 @@ module Textus
       )
     end
   end
+
+  class CursorExpired < Error
+    attr_reader :requested, :min_available
+
+    def initialize(requested:, min_available:)
+      @requested = requested
+      @min_available = min_available
+      super(
+        "cursor_expired",
+        "audit cursor expired: requested seq=#{requested} but oldest available is #{min_available}; " \
+        "call `textus boot` to re-orient and resume from latest_seq",
+        details: { "requested" => requested, "min_available" => min_available },
+        hint: "call `textus boot` to get the current latest_seq and resume from there",
+      )
+    end
+  end
 end

data/lib/textus/infra/audit_log.rb

@@ -1,11 +1,18 @@
+require "fileutils"
 require "json"
 require "time"
 
 module Textus
   module Infra
     class AuditLog
-      def initialize(root)
-        @path = File.join(root, "audit.log")
+      DEFAULT_MAX_SIZE = 10_485_760
+      DEFAULT_KEEP = 5
+
+      def initialize(root, max_size: DEFAULT_MAX_SIZE, keep: DEFAULT_KEEP)
+        @root     = root
+        @path     = File.join(root, "audit.log")
+        @max_size = max_size
+        @keep     = keep
       end
 
       def last_writer_for(key)
@@ -23,27 +30,38 @@ module Textus
         last_role
       end
 
-      def append(role:, verb:, key:, etag_before:, etag_after:, extras: nil)
-        row = {
-          "ts" => Time.now.utc.iso8601,
-          "role" => role,
-          "verb" => verb,
-          "key" => key,
-          "etag_before" => etag_before,
-          "etag_after" => etag_after,
-        }
+      def latest_seq
+        return scan_max_seq(@path) if File.exist?(@path) && File.size(@path).positive?
 
-        if extras.is_a?(Hash) && !extras.empty?
-          extras = extras.dup
-          %w[from_key to_key uid].each do |k|
-            row[k] = extras.delete(k) if extras.key?(k)
+        # Active log is empty/missing — consult the most recent rotated file's sidecar.
+        meta = read_meta(1)
+        return meta["max_seq"] if meta
+
+        0
+      end
+
+      def min_available_seq
+        rotated_metas = (1..@keep).map { |n| read_meta(n) }.compact
+        if rotated_metas.any?
+          rotated_metas.map { |m| m["min_seq"] }.min
+        elsif File.exist?(@path)
+          File.foreach(@path) do |line|
+            parsed = parse_row(line.chomp)
+            return parsed["seq"] if parsed && parsed["seq"]
           end
-          row["extras"] = extras unless extras.empty?
+          nil
         end
+      end
 
+      def append(role:, verb:, key:, etag_before:, etag_after:, extras: nil)
         File.open(@path, File::WRONLY | File::APPEND | File::CREAT, 0o644) do |f|
           f.flock(File::LOCK_EX)
+          next_seq = current_max_seq_unlocked + 1
+          row = assemble_row(next_seq, { role: role, verb: verb, key: key,
+                                         etag_before: etag_before, etag_after: etag_after }, extras)
           f.write(JSON.generate(row) + "\n")
+          f.flush
+          rotate!(f) if f.size > @max_size
         end
       end
 
@@ -63,6 +81,98 @@ module Textus
 
       private
 
+      # Caller holds the flock. Returns the highest seq across the active log,
+      # OR the most-recent rotated file's max_seq if the active log is empty.
+      def current_max_seq_unlocked
+        return scan_max_seq(@path) if File.exist?(@path) && File.size(@path).positive?
+
+        meta = read_meta(1)
+        meta ? meta["max_seq"] : 0
+      end
+
+      def scan_max_seq(file)
+        last = 0
+        File.foreach(file) do |line|
+          parsed = parse_row(line.chomp)
+          last = parsed["seq"] if parsed && parsed["seq"].is_a?(Integer)
+        end
+        last
+      end
+
+      def scan_seq_range(file)
+        min = nil
+        max = 0
+        File.foreach(file) do |line|
+          parsed = parse_row(line.chomp)
+          next unless parsed && parsed["seq"]
+
+          min ||= parsed["seq"]
+          max = parsed["seq"]
+        end
+        [min, max]
+      end
+
+      def read_meta(n)
+        path = File.join(@root, "audit.log.#{n}.meta.json")
+        return nil unless File.exist?(path)
+
+        JSON.parse(File.read(path))
+      rescue JSON::ParserError
+        nil
+      end
+
+      def assemble_row(seq, fields, extras = nil)
+        row = {
+          "seq" => seq,
+          "ts" => Time.now.utc.iso8601,
+          "role" => fields[:role],
+          "verb" => fields[:verb],
+          "key" => fields[:key],
+          "etag_before" => fields[:etag_before],
+          "etag_after" => fields[:etag_after],
+        }
+
+        if extras.is_a?(Hash) && !extras.empty?
+          extras = extras.dup
+          %w[from_key to_key uid].each do |k|
+            row[k] = extras.delete(k) if extras.key?(k)
+          end
+          row["extras"] = extras unless extras.empty?
+        end
+
+        row
+      end
+
+      # Called inside the flock, after a successful write that pushed size over max.
+      # Renames audit.log → audit.log.1 (shifting older files), writes sidecar meta.
+      def rotate!(open_file)
+        open_file.flush
+        min_seq, max_seq = scan_seq_range(@path)
+        meta = { "min_seq" => min_seq, "max_seq" => max_seq, "rotated_at" => Time.now.utc.iso8601 }
+
+        # Drop the file that would be shifted past @keep.
+        oldest      = File.join(@root, "audit.log.#{@keep}")
+        oldest_meta = File.join(@root, "audit.log.#{@keep}.meta.json")
+        FileUtils.rm_f(oldest)
+        FileUtils.rm_f(oldest_meta)
+
+        # Shift .N → .(N+1) for N = keep-1 down to 1.
+        (@keep - 1).downto(1) do |n|
+          src      = File.join(@root, "audit.log.#{n}")
+          dst      = File.join(@root, "audit.log.#{n + 1}")
+          File.rename(src, dst) if File.exist?(src)
+
+          src_meta = File.join(@root, "audit.log.#{n}.meta.json")
+          dst_meta = File.join(@root, "audit.log.#{n + 1}.meta.json")
+          File.rename(src_meta, dst_meta) if File.exist?(src_meta)
+        end
+
+        # Active log → .1
+        File.rename(@path, File.join(@root, "audit.log.1"))
+        File.write(File.join(@root, "audit.log.1.meta.json"), JSON.generate(meta) + "\n")
+        # Next append will create a fresh audit.log via File::CREAT.
+      end
+
       def parse_row(line)
         return nil if line.empty?
 

data/lib/textus/manifest/entry/base.rb

@@ -2,10 +2,10 @@ module Textus
   class Manifest
     class Entry
       class Base < Entry
-        attr_reader :raw, :key, :path, :zone, :schema, :owner, :format, :manifest
+        attr_reader :raw, :key, :path, :zone, :schema, :owner, :format, :manifest, :publish_to
 
         # rubocop:disable Metrics/ParameterLists, Lint/MissingSuper
-        def initialize(manifest:, raw:, key:, path:, zone:, schema:, owner:, format:)
+        def initialize(manifest:, raw:, key:, path:, zone:, schema:, owner:, format:, publish_to: [])
           @manifest = manifest
           @raw = raw
           @key = key
@@ -14,11 +14,10 @@ module Textus
           @schema = schema
           @owner = owner
           @format = format
+          @publish_to = Array(publish_to)
         end
         # rubocop:enable Metrics/ParameterLists, Lint/MissingSuper
 
-        def kind = self.class.name.split("::").last.downcase.to_sym
-
         def zone_writers
           @manifest.zone_writers(@zone)
         rescue UsageError => e
@@ -32,6 +31,44 @@ module Textus
         def derived? = false
         def intake?  = false
         def leaf?    = false
+
+        # Nil stubs for cross-cutting optional attrs. Subclasses override the
+        # ones they own. Validators and serializers can call these directly
+        # without `respond_to?` guards.
+        def template       = nil
+        def inject_boot    = false # rubocop:disable Naming/PredicateMethod
+        def events         = {}
+        def publish_each   = nil
+        def index_filename = nil
+
+        PublishContext = Struct.new(
+          :repo_root, :manifest, :file_store, :root, :store, :ctx, :bus, :hook_context,
+          :reader, :emit, # callables: reader.call(key) → envelope; emit.call(event, **payload)
+          keyword_init: true
+        )
+
+        # Subclasses override to customize publish behavior.
+        # Default: copy the stored file to each publish_to target.
+        # Returns: { kind: :built|:leaves, value: ... } to be accumulated by
+        # Publish#call, or nil to skip.
+        def publish_via(pctx, prefix: nil) # rubocop:disable Lint/UnusedMethodArgument
+          return nil if Array(publish_to).empty?
+
+          source_path = pctx.manifest.resolver.resolve(@key).path
+          envelope = pctx.reader.call(@key)
+
+          publish_to.each do |rel|
+            target_abs = File.join(pctx.repo_root, rel)
+            Textus::Infra::Publisher.publish(source: source_path, target: target_abs, store_root: pctx.root)
+            pctx.emit.call(:file_published,
+                           key: @key,
+                           envelope: envelope,
+                           source: source_path,
+                           target: target_abs)
+          end
+
+          { kind: :built, value: { "key" => @key, "path" => source_path, "published_to" => publish_to } }
+        end
       end
     end
   end

data/lib/textus/manifest/entry/derived.rb

@@ -5,20 +5,56 @@ module Textus
         Projection = Data.define(:select, :pluck, :sort_by, :transform)
         External   = Data.define(:sources, :runner)
 
-        attr_reader :source, :template, :inject_intro, :publish_to, :events
+        attr_reader :source, :template, :inject_boot, :events
 
-        def initialize(source:, template: nil, inject_intro: false, publish_to: [], events: {}, **rest)
+        def initialize(source:, template: nil, inject_boot: false, events: {}, **rest)
           super(**rest)
           @source = source
           @template = template
-          @inject_intro = inject_intro
-          @publish_to = Array(publish_to)
+          @inject_boot = inject_boot
           @events = events || {}
         end
 
         def derived? = true
         def projection? = @source.is_a?(Projection)
         def external?   = @source.is_a?(External)
+
+        def publish_via(pctx, prefix: nil) # rubocop:disable Lint/UnusedMethodArgument
+          return nil unless in_generator_zone?
+
+          target_path = Textus::Application::Writes::Materializer.new(
+            ctx: pctx.ctx, manifest: pctx.manifest, file_store: pctx.file_store,
+            bus: pctx.bus, root: pctx.root, store: pctx.store
+          ).run(self)
+
+          envelope = pctx.reader.call(@key)
+          Array(publish_to).each do |rel|
+            target_abs = File.join(pctx.repo_root, rel)
+            Textus::Infra::Publisher.publish(source: target_path, target: target_abs, store_root: pctx.root)
+            pctx.emit.call(:file_published, key: @key, envelope: envelope, source: target_path, target: target_abs)
+          end
+
+          src = @source
+          selects = src.is_a?(Projection) ? Array(src.select).compact : []
+          pctx.emit.call(:build_completed, key: @key, envelope: envelope, sources: selects)
+
+          { kind: :built, value: { "key" => @key, "path" => target_path, "published_to" => publish_to } }
+        end
+
+        KIND = :derived
+
+        def self.from_raw(common, raw)
+          source = Parser.parse_source(raw, common[:key])
+          new(
+            source: source,
+            template: raw["template"],
+            inject_boot: raw["inject_boot"] == true,
+            events: raw["events"] || {},
+            **common,
+          )
+        end
+
+        Entry::REGISTRY[KIND] = self
       end
     end
   end

data/lib/textus/manifest/entry/intake.rb

@@ -2,17 +2,29 @@ module Textus
   class Manifest
     class Entry
       class Intake < Base
-        attr_reader :handler, :config, :events, :publish_to
+        attr_reader :handler, :config, :events
 
         def initialize(handler:, config: {}, events: {}, **rest)
           super(**rest)
           @handler = handler
           @config = config || {}
           @events = events || {}
-          @publish_to = []
         end
 
-        def intake? = true
+        def intake?  = true
+        def nested?  = !!@raw["nested"]
+
+        KIND = :intake
+
+        def self.from_raw(common, raw)
+          intake = raw["intake"] || {}
+          handler = intake["handler"] || raw["intake_handler"] or
+            raise UsageError.new("intake entry '#{common[:key]}' missing handler")
+          config = intake["config"] || raw["intake_config"] || {}
+          new(handler: handler, config: config, events: raw["events"] || {}, **common)
+        end
+
+        Entry::REGISTRY[KIND] = self
       end
     end
   end

data/lib/textus/manifest/entry/leaf.rb

@@ -2,14 +2,15 @@ module Textus
   class Manifest
     class Entry
       class Leaf < Base
-        attr_reader :publish_to
+        KIND = :leaf
 
-        def initialize(publish_to: [], **rest)
-          super(**rest)
-          @publish_to = Array(publish_to)
+        def leaf? = true
+
+        def self.from_raw(common, _raw)
+          new(**common)
         end
 
-        def leaf? = true
+        Entry::REGISTRY[KIND] = self
       end
     end
   end

data/lib/textus/manifest/entry/nested.rb

@@ -7,13 +7,12 @@ module Textus
         PUBLISH_EACH_VARS   = Validators::PublishEach::KNOWN_VARS
         PUBLISH_EACH_VAR_RE = Validators::PublishEach::VAR_RE
 
-        attr_reader :index_filename, :publish_each, :publish_to
+        attr_reader :index_filename, :publish_each
 
-        def initialize(index_filename: nil, publish_each: nil, publish_to: [], **rest)
+        def initialize(index_filename: nil, publish_each: nil, **rest)
           super(**rest)
           @index_filename = index_filename
           @publish_each = publish_each
-          @publish_to = Array(publish_to)
         end
 
         def nested? = true
@@ -33,6 +32,46 @@ module Textus
           vars = { "leaf" => leaf, "basename" => basename, "key" => full_key, "ext" => ext }
           @publish_each.gsub(PUBLISH_EACH_VAR_RE) { vars.fetch(::Regexp.last_match(1)) }
         end
+
+        def publish_via(pctx, prefix: nil)
+          return nil if @publish_each.nil?
+
+          leaves = []
+          @manifest.resolver.enumerate(prefix: @key).each do |row|
+            next unless row[:manifest_entry].equal?(self)
+            next if prefix && !row[:key].start_with?(prefix) && row[:key] != prefix
+
+            target_rel = publish_target_for(row[:key])
+            target_abs = File.expand_path(File.join(pctx.repo_root, target_rel))
+            unless target_abs.start_with?(File.expand_path(pctx.repo_root) + File::SEPARATOR)
+              raise Textus::PublishError.new(
+                "entry '#{@key}': publish_each target '#{target_rel}' for key '#{row[:key]}' escapes repo root",
+              )
+            end
+
+            Textus::Infra::Publisher.publish(source: row[:path], target: target_abs, store_root: pctx.root)
+            pctx.emit.call(:file_published,
+                           key: row[:key],
+                           envelope: pctx.reader.call(row[:key]),
+                           source: row[:path],
+                           target: target_abs)
+            leaves << { "key" => row[:key], "source" => row[:path], "target" => target_abs }
+          end
+
+          { kind: :leaves, value: leaves }
+        end
+
+        KIND = :nested
+
+        def self.from_raw(common, raw)
+          new(
+            index_filename: raw["index_filename"],
+            publish_each: raw["publish_each"],
+            **common,
+          )
+        end
+
+        Entry::REGISTRY[KIND] = self
       end
     end
   end

data/lib/textus/manifest/entry/parser.rb

@@ -9,7 +9,7 @@ module Textus
           path = raw["path"] or raise UsageError.new("manifest entry '#{key}' missing path")
           zone = raw["zone"] or raise UsageError.new("manifest entry '#{key}' missing zone")
 
-          raw_kind = raw["kind"] or raise BadManifest.new("entry '#{key}' missing required `kind:` (leaf|nested|derived|intake)")
+          raw_kind = raw["kind"] or raise BadManifest.new("entry '#{key}' missing required `kind:` (#{Entry::REGISTRY.keys.join("|")})")
           kind = raw_kind.to_sym
           format = resolve_format(raw, path)
 
@@ -17,49 +17,13 @@ module Textus
             manifest: manifest, raw: raw,
             key: key, path: path, zone: zone,
             schema: raw["schema"], owner: raw["owner"],
-            format: format
+            format: format,
+            publish_to: raw["publish_to"]
           }
 
-          case kind
-          when :leaf    then build_leaf(common, raw)
-          when :nested  then build_nested(common, raw)
-          when :derived then build_derived(common, raw, key)
-          when :intake  then build_intake(common, raw, key)
-          else raise BadManifest.new("entry '#{key}': unknown kind: #{kind.inspect}")
-          end
-        end
-
-        def self.build_leaf(common, raw)
-          Leaf.new(publish_to: raw["publish_to"], **common)
-        end
-
-        def self.build_nested(common, raw)
-          Nested.new(
-            index_filename: raw["index_filename"],
-            publish_each: raw["publish_each"],
-            publish_to: raw["publish_to"],
-            **common,
-          )
-        end
-
-        def self.build_derived(common, raw, key)
-          source = parse_source(raw, key)
-          Derived.new(
-            source: source,
-            template: raw["template"],
-            inject_intro: raw["inject_intro"] == true,
-            publish_to: raw["publish_to"],
-            events: raw["events"] || {},
-            **common,
-          )
-        end
-
-        def self.build_intake(common, raw, key)
-          intake = raw["intake"] || {}
-          handler = intake["handler"] || raw["intake_handler"] or
-            raise UsageError.new("intake entry '#{key}' missing handler")
-          config = intake["config"] || raw["intake_config"] || {}
-          Intake.new(handler: handler, config: config, events: raw["events"] || {}, **common)
+          klass = Entry::REGISTRY[kind] or
+            raise BadManifest.new("entry '#{key}': unknown kind: #{kind.inspect} (known: #{Entry::REGISTRY.keys.join(", ")})")
+          klass.from_raw(common, raw)
         end
 
         def self.parse_source(raw, key)
@@ -73,14 +37,14 @@ module Textus
           end
 
           if compute["kind"] == "projection"
-            Derived::Projection.new(
+            Entry::Derived::Projection.new(
               select: compute["select"],
               pluck: compute["pluck"],
               sort_by: compute["sort_by"],
               transform: compute["transform"],
             )
           else
-            Derived::External.new(sources: compute["sources"], runner: compute["runner"])
+            Entry::Derived::External.new(sources: compute["sources"], runner: compute["runner"])
           end
         end
 

data/lib/textus/manifest/entry/validators/events.rb

@@ -5,7 +5,7 @@ module Textus
         module Events
           def self.call(entry)
             pubsub_events = Textus::Hooks::Bus::EVENTS.select { |_, s| s[:mode] == :pubsub }.keys
-            events = entry.respond_to?(:events) ? entry.events : {}
+            events = entry.events
             events.each_key do |evt|
               next if pubsub_events.include?(evt.to_sym)
 

data/lib/textus/manifest/entry/validators/format_matrix.rb

@@ -14,12 +14,13 @@ module Textus
               raise UsageError.new("entry '#{entry.key}': text format must not declare a schema")
             end
 
-            has_template = entry.respond_to?(:template) && !entry.template.nil?
-            is_external   = entry.derived? && entry.external?
-            return unless entry.in_generator_zone? && !has_template && !is_external &&
+            has_template = !entry.template.nil?
+            is_external  = entry.derived? && entry.external?
+            is_intake    = entry.intake?
+            return unless entry.in_generator_zone? && !has_template && !is_external && !is_intake &&
                           %w[markdown text].include?(entry.format) && !entry.nested?
 
-            raise UsageError.new("entry '#{entry.key}': derived #{entry.format} entries require a template")
+            raise UsageError.new("entry '#{entry.key}': #{entry.format} entries in a generator zone require a template")
           end
         end
       end

data/lib/textus/manifest/entry/validators/index_filename.rb

@@ -4,7 +4,8 @@ module Textus
       module Validators
         module IndexFilename
           def self.call(entry)
-            index_filename = entry.respond_to?(:index_filename) ? entry.index_filename : entry.raw["index_filename"]
+            # Use raw to detect misuse on non-nested entries (typed attr stubs nil on Base).
+            index_filename = entry.nested? ? entry.index_filename : entry.raw["index_filename"]
             return if index_filename.nil?
 
             check_shape!(entry, index_filename)

data/lib/textus/manifest/entry/validators/inject_boot.rb

@@ -0,0 +1,19 @@
+module Textus
+  class Manifest
+    class Entry
+      module Validators
+        module InjectBoot
+          def self.call(entry)
+            return unless entry.inject_boot
+
+            raise UsageError.new("entry '#{entry.key}': inject_boot: is only valid on derived entries") unless entry.in_generator_zone?
+
+            return unless entry.template.nil?
+
+            raise UsageError.new("entry '#{entry.key}': inject_boot: requires a template:")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/textus/manifest/entry/validators/publish_each.rb

@@ -7,13 +7,14 @@ module Textus
           VAR_RE = /\{([a-z]+)\}/
           REQUIRED_DISCRIMINATOR_VARS = %w[leaf basename key].freeze
 
-          def self.call(entry) # rubocop:disable Metrics/AbcSize
-            publish_each = entry.respond_to?(:publish_each) ? entry.publish_each : entry.raw["publish_each"]
+          def self.call(entry)
+            # Use raw to detect misuse on non-nested entries (typed attr stubs nil on Base).
+            publish_each = entry.nested? ? entry.publish_each : entry.raw["publish_each"]
             return if publish_each.nil?
 
             raise UsageError.new("entry '#{entry.key}': publish_each requires nested: true") unless entry.nested?
 
-            publish_to = entry.respond_to?(:publish_to) ? entry.publish_to : Array(entry.raw["publish_to"])
+            publish_to = entry.publish_to
             raise UsageError.new("entry '#{entry.key}': publish_to and publish_each are mutually exclusive") unless publish_to.empty?
             raise UsageError.new("entry '#{entry.key}': publish_each must be a string") unless publish_each.is_a?(String)
 

data/lib/textus/manifest/entry/validators.rb

@@ -5,7 +5,7 @@ module Textus
         REGISTERED = [
           Events,
           PublishEach,
-          InjectIntro,
+          InjectBoot,
           IndexFilename,
           FormatMatrix,
         ].freeze