textus 0.55.2 → 0.56.0

data/lib/textus/protocol/builder/infrastructure.rb

@@ -0,0 +1,17 @@
+module Textus
+  module Protocol
+    class Builder
+      class Infrastructure
+        Components = Data.define(:database, :file_system, :store, :layout)
+
+        def call(root)
+          layout = Layout.new(root)
+          database = Infra::Database.new(layout:).setup!
+          file_system = Infra::FileSystem.new
+          store = Infra::Store.new(database:, file_system:)
+          Components.new(database:, file_system:, store:, layout:)
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/builder/runtime.rb

@@ -0,0 +1,46 @@
+module Textus
+  module Protocol
+    class Builder
+      class Runtime
+        Components = Data.define(:store_engine, :gate, :ttl_evaluator, :container)
+
+        def call(infra:, domain:, root:)
+          event_emitter = Textus::Workflow::EventEmitter.new(
+            infra.store.workflow_queue,
+            registry: domain.workflow_registry,
+          )
+
+          store_engine = StoreEngine.new(
+            file_system: infra.file_system, manifest: domain.manifest,
+            layout: infra.layout, store: infra.store,
+            event_emitter:, schemas: domain.schemas
+          )
+
+          ttl_evaluator = Freshness::TtlEvaluator.new(
+            manifest: domain.manifest,
+            file_stat: infra.file_system,
+            clock: Infra::Clock.new,
+          )
+
+          container = Container.new(
+            store: infra.store, layout: infra.layout,
+            manifest: domain.manifest, schemas: domain.schemas,
+            workflows: domain.workflows,
+            store_engine: store_engine,
+            root:, ttl_evaluator:
+          )
+
+          gate = Gate.new(
+            manifest: domain.manifest,
+            resolver: domain.manifest.resolver,
+            container:,
+            store_engine:,
+          )
+
+          container.gate = gate
+          Components.new(store_engine:, gate:, ttl_evaluator:, container:)
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/builder.rb

@@ -0,0 +1,11 @@
+module Textus
+  module Protocol
+    class Builder
+      def call(root)
+        infra = Infrastructure.new.call(root)
+        domain = Domain.new.call(root, layout: infra.layout)
+        Runtime.new.call(infra:, domain:, root:).container
+      end
+    end
+  end
+end

data/lib/textus/protocol/container.rb

@@ -0,0 +1,22 @@
+module Textus
+  module Protocol
+    class Container
+      attr_accessor :gate
+      attr_reader :store, :layout, :manifest, :schemas, :workflows, :workflow_registry, :root, :ttl_evaluator, :store_engine
+
+      def initialize(store:, layout:, manifest:, schemas:, workflows:,
+                     root: nil, gate: nil, ttl_evaluator: nil, store_engine: nil)
+        @store = store
+        @layout = layout
+        @manifest = manifest
+        @schemas = schemas
+        @workflows = workflows
+        @workflow_registry = workflows ? Textus::Workflow::Registry.new(workflows) : nil
+        @root = root
+        @gate = gate
+        @ttl_evaluator = ttl_evaluator
+        @store_engine = store_engine
+      end
+    end
+  end
+end

data/lib/textus/protocol/diff.rb

@@ -0,0 +1,170 @@
+module Textus
+  module Protocol
+    module Diff
+      module_function
+
+      def body(a_text, b_text)
+        a_lines = (a_text || "").lines.map(&:chomp)
+        b_lines = (b_text || "").lines.map(&:chomp)
+        hunks = build_hunks(myers_diff(a_lines, b_lines), a_lines, b_lines)
+        { "type" => "body", "hunks" => hunks, "stats" => stats(hunks) }
+      end
+
+      def meta(a_hash, b_hash)
+        a = a_hash || {}
+        b = b_hash || {}
+        all_keys = (a.keys + b.keys).uniq.sort
+        changes = all_keys.filter_map do |k|
+          av = a[k]
+          bv = b[k]
+          next if av == bv
+
+          { "key" => k.to_s, "old" => av, "new" => bv }
+        end
+        { "type" => "meta", "changes" => changes, "stats" => { "changed" => changes.size } }
+      end
+
+      def schema(a_schema, b_schema)
+        meta(a_schema, b_schema).merge("type" => "schema")
+      end
+
+      def summary(diff_result)
+        parts = []
+        parts << "#{diff_result.dig("body", "stats", "additions") || 0}+" if diff_result["body"]
+        parts << "#{diff_result.dig("body", "stats", "deletions") || 0}-" if diff_result["body"]
+        parts << "#{diff_result.dig("meta", "stats", "changed") || 0}~" if diff_result["meta"]
+        parts.join(" ")
+      end
+
+      def myers_diff(a_lines, b_lines)
+        a_count = a_lines.size
+        b_count = b_lines.size
+        max_d = a_count + b_count
+        v = Hash.new(0)
+        v[1] = 0
+        trace = []
+
+        (0..max_d).each do |d|
+          (-d..d).step(2).each do |k|
+            x = if k == -d || (k != d && v[k - 1] < v[k + 1])
+                  v[k + 1]
+                else
+                  v[k - 1] + 1
+                end
+            y = x - k
+            while x < a_count && y < b_count && a_lines[x] == b_lines[y]
+              x += 1
+              y += 1
+            end
+            v[k] = x
+            return backtrack(trace, x, y) if x >= a_count && y >= b_count
+          end
+          trace << v.dup
+        end
+        []
+      end
+
+      def backtrack(trace, x_pos, y_pos)
+        script = []
+        d = trace.size - 1
+        return script if d.negative?
+
+        k = x_pos - y_pos
+        while d.positive?
+          prev_v = trace[d - 1]
+          prev_k = if k == -d || (k != d && (prev_v[k - 1] || -1) < (prev_v[k + 1] || -1))
+                     k + 1
+                   else
+                     k - 1
+                   end
+          prev_x = prev_v[prev_k] || 0
+          prev_y = prev_x - prev_k
+
+          while x_pos > prev_x && y_pos > prev_y
+            script.unshift([:eq, x_pos - 1, y_pos - 1])
+            x_pos -= 1
+            y_pos -= 1
+          end
+
+          if x_pos > prev_x
+            script.unshift([:del, x_pos - 1, nil])
+            x_pos -= 1
+          elsif y_pos > prev_y
+            script.unshift([:ins, nil, y_pos - 1])
+            y_pos -= 1
+          end
+
+          d -= 1
+          k = prev_k
+          x_pos = prev_x
+          y_pos = prev_y
+        end
+
+        script
+      end
+
+      def build_hunks(script, a_lines, b_lines)
+        return [] if script.empty?
+
+        hunks = []
+        i = 0
+        while i < script.size
+          next_context = find_next_change(script, i)
+          break unless next_context
+
+          start = [next_context - 3, 0].max
+          hunk_end_script = find_hunk_end(script, next_context)
+          hunk_end = [hunk_end_script + 3, script.size - 1].min
+
+          a_start = nil
+          b_start = nil
+          a_lines_hunk = []
+          b_lines_hunk = []
+
+          (start..hunk_end).each do |j|
+            op, ai, bi = script[j]
+            a_start ||= ai if ai
+            b_start ||= bi if bi
+
+            case op
+            when :eq
+              a_lines_hunk << { "type" => "context", "line" => a_lines[ai] }
+              b_lines_hunk << { "type" => "context", "line" => b_lines[bi] }
+            when :del
+              a_lines_hunk << { "type" => "deletion", "line" => a_lines[ai] }
+            when :ins
+              b_lines_hunk << { "type" => "addition", "line" => b_lines[bi] }
+            end
+          end
+
+          hunks << {
+            "a_start" => a_start, "b_start" => b_start,
+            "a_lines" => a_lines_hunk, "b_lines" => b_lines_hunk
+          }
+
+          i = hunk_end + 1
+        end
+
+        hunks
+      end
+
+      def find_next_change(script, from)
+        (from...script.size).find { |i| script[i][0] != :eq }
+      end
+
+      def find_hunk_end(script, from)
+        last_change = from
+        (from...[script.size, from + 6].min).each do |i|
+          last_change = i if script[i][0] != :eq
+        end
+        last_change
+      end
+
+      def stats(hunks)
+        additions = hunks.sum { |h| h["b_lines"].count { |l| l["type"] == "addition" } }
+        deletions = hunks.sum { |h| h["a_lines"].count { |l| l["type"] == "deletion" } }
+        { "additions" => additions, "deletions" => deletions, "hunks" => hunks.size }
+      end
+    end
+  end
+end

data/lib/textus/protocol/entry_constraint.rb

@@ -0,0 +1,54 @@
+module Textus
+  module Protocol
+    module EntryConstraint
+      module_function
+
+      def validate_naming!(key, mentry)
+        naming = mentry.naming
+        return unless naming
+        return unless key.start_with?("#{mentry.key}.")
+
+        last = key.split(".").last
+
+        case naming
+        when "sequential"
+          return if last.match?(/\A\d{4}-[a-z][a-z0-9-]*\z/)
+
+          raise UsageError.new(
+            "naming violation: '#{key}' last segment must match NNNN-slug " \
+            "(entry '#{mentry.key}' has naming: sequential)",
+          )
+        when "dated"
+          return if last.match?(/\A\d{4}-\d{2}-\d{2}-[a-z][a-z0-9-]*\z/)
+
+          raise UsageError.new(
+            "naming violation: '#{key}' last segment must match YYYY-MM-DD-slug " \
+            "(entry '#{mentry.key}' has naming: dated)",
+          )
+        end
+      end
+
+      def validate_schema!(payload, mentry, schemas:)
+        schema_name = mentry.schema
+        return unless schema_name && schemas
+
+        schema = schemas.fetch_or_nil(schema_name)
+        return unless schema
+        return if mentry.lane == "scratchpad"
+
+        schema.validate!(payload.meta || {})
+      rescue Textus::SchemaViolation
+        raise if mentry.lane != "scratchpad"
+      end
+
+      def validate_format!(payload, mentry, format: nil)
+        fmt_name = format || mentry.format
+        fmt = Textus::Protocol::Format.for(fmt_name)
+        fmt.validate_raw_entry!(
+          { "_meta" => payload.meta, "content" => payload.content },
+          mentry.lane,
+        )
+      end
+    end
+  end
+end

data/lib/textus/protocol/envelope.rb

@@ -0,0 +1,70 @@
+require "dry-struct"
+
+module Textus
+  module Protocol
+    class Envelope < Dry::Struct
+      attribute :protocol,   Textus::Value::Types::String
+      attribute :key,        Textus::Value::Types::String
+      attribute :lane,       Textus::Value::Types::String
+      attribute :path,       Textus::Value::Types::String
+      attribute :format,     Textus::Value::Types::FormatName
+      attribute :etag,       Textus::Value::Types::String
+      attribute :uid,        Textus::Value::Types::String.optional
+      attribute :sources,    Textus::Value::Types::Array.of(Textus::Value::Types::Any).optional
+      attribute :schema_ref, Textus::Value::Types::String.optional
+      attribute :meta,       Textus::Value::Types::Hash.default({}.freeze)
+      attribute :body,       Textus::Value::Types::String.optional
+      attribute :content,    Textus::Value::Types::Any.optional
+      attribute :freshness,  Textus::Value::Types::Any.optional
+
+      def self.build(key:, mentry:, path:, meta:, body:, etag:, content: nil, freshness: nil)
+        new(
+          protocol: Textus::PROTOCOL,
+          key:,
+          lane: mentry.lane,
+          path:,
+          format: mentry.format,
+          uid: extract_uid(meta),
+          sources: extract_sources(meta),
+          etag:,
+          schema_ref: mentry.schema,
+          meta: meta || {},
+          body:,
+          content:,
+          freshness:,
+        )
+      end
+
+      def with(**attrs) = self.class.new(to_h.merge(attrs))
+
+      def to_h_for_wire
+        h = {
+          "protocol" => protocol, "key" => key, "lane" => lane,
+          "path" => path, "format" => format, "_meta" => meta,
+          "body" => body, "etag" => etag, "schema_ref" => schema_ref, "uid" => uid
+        }
+        h["sources"] = sources if sources
+        h["content"] = content unless content.nil?
+        freshness&.to_h_for_wire&.each { |k, v| h[k] = v }
+        h
+      end
+
+      def stale? = freshness.is_a?(Hash) && freshness["stale"] == true
+
+      def fetching? = freshness.is_a?(Hash) && freshness["fetching"] == true
+
+      def self.extract_uid(meta)
+        v = meta.is_a?(Hash) ? meta["uid"] : nil
+        v.is_a?(String) ? v : nil
+      end
+
+      def self.extract_sources(meta)
+        v = meta.is_a?(Hash) ? meta["sources"] : nil
+        return nil unless v.is_a?(Array) && !v.empty?
+
+        valid = v.select { |s| s.is_a?(String) || (s.is_a?(Hash) && s["key"].is_a?(String)) }
+        valid.empty? ? nil : valid
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/base.rb

@@ -0,0 +1,80 @@
+module Textus
+  module Protocol
+    module Format
+      class Base
+        def self.parse(_raw, path: nil)
+          _ = path
+          raise NotImplementedError.new("#{name}.parse not implemented")
+        end
+
+        def self.serialize(meta: {}, body: "", content: nil)
+          _ = meta
+          _ = body
+          _ = content
+          raise NotImplementedError.new("#{name}.serialize not implemented")
+        end
+
+        def self.extensions
+          raise NotImplementedError.new("#{name}.extensions not implemented")
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["_meta"] || {})
+        end
+
+        def self.nested_glob
+          raise NotImplementedError.new("#{name}.nested_glob not implemented")
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("#{format_name} nested path must not have an extension")
+          end
+
+          return if extensions.include?(ext)
+
+          raise UsageError.new("#{format_name} format requires '#{extensions.join("' or '")}' path (got #{ext.inspect})")
+        end
+
+        def self.enforce_name_match!(path, meta)
+          return unless meta.is_a?(Hash) && meta["name"]
+
+          ext = extensions.first
+          basename = File.basename(path, ext)
+          return if meta["name"] == basename
+
+          raise BadFrontmatter.new(path, "name '#{meta["name"]}' does not match basename '#{basename}'")
+        end
+
+        def self.rewrite_name(path, basename) # rubocop:disable Naming/PredicateMethod
+          raw = File.binread(path)
+          parsed = parse(raw, path: path)
+          meta = parsed["_meta"] || {}
+          return false unless meta.is_a?(Hash) && meta["name"].is_a?(String) && meta["name"] != basename
+
+          new_meta = meta.merge("name" => basename)
+          File.binwrite(path, serialize(meta: new_meta, body: parsed["body"] || "", content: parsed["content"]))
+          true
+        end
+
+        def self.format_name
+          name.split("::").last.downcase
+        end
+
+        def self.validate_raw_entry!(_parsed, _lane); end
+
+        def self.nested_ext(_segment = nil) = extensions.first
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          _ = path
+          _ = content
+          bytes = serialize(meta: meta || {}, body: body.to_s)
+          [bytes, meta, body.to_s, nil]
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/bash.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Bash < Script
+        def self.extensions = [".sh"]
+        def self.nested_ext(*) = ".sh"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/javascript.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Javascript < Script
+        def self.extensions = [".js"]
+        def self.nested_ext(*) = ".js"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/json.rb

@@ -0,0 +1,82 @@
+require "json"
+
+module Textus
+  module Protocol
+    module Format
+      class Json < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+
+          begin
+            parsed = ::JSON.parse(raw)
+          rescue ::JSON::ParserError => e
+            raise BadFrontmatter.new(path, "JSON parse failed: #{e.message}")
+          end
+          raise BadFrontmatter.new(path, "JSON top-level must be an object") unless parsed.is_a?(Hash)
+
+          meta = parsed["_meta"]
+          fm = meta.is_a?(Hash) ? meta : {}
+          content_without_meta = parsed.except("_meta")
+          { "_meta" => fm, "body" => raw, "content" => content_without_meta }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          if content.is_a?(Hash)
+            on_disk = meta && !meta.empty? ? { "_meta" => meta }.merge(content) : content
+            out = ::JSON.pretty_generate(on_disk)
+            out += "\n" unless out.end_with?("\n")
+            out
+          elsif body && !body.to_s.empty?
+            b = body.to_s
+            b += "\n" unless b.end_with?("\n")
+            b
+          else
+            raise UsageError.new("json serialize requires :content or :body")
+          end
+        end
+
+        def self.validate_against(schema, parsed)
+          schema.validate!(parsed["content"] || {})
+        end
+
+        def self.extensions = [".json"]
+        def self.nested_glob = "**/*.json"
+
+        def self.serialize_for_put(meta:, body:, content:, path:)
+          raise UsageError.new("put for json requires content: or body:") if content.nil? && (body.nil? || body.to_s.empty?)
+
+          if content.nil?
+            begin
+              parsed = parse(body.to_s, path: path)
+            rescue BadFrontmatter => e
+              raise BadContent.new(path, "bad_content: #{e.message}")
+            end
+            [body.to_s, parsed["_meta"], body.to_s, parsed["content"]]
+          else
+            bytes = serialize(meta: meta, body: "", content: content)
+            [bytes, meta, bytes, content]
+          end
+        end
+
+        def self.validate_path_extension(path, nested)
+          ext = File.extname(path)
+          if nested
+            return if ext == ""
+
+            raise UsageError.new("nested json path must not have an extension")
+          end
+
+          return if ext == ".json"
+
+          raise UsageError.new("json format requires '.json' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: nil, content: data["content"] || data }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/markdown.rb

@@ -0,0 +1,53 @@
+require "yaml"
+
+module Textus
+  module Protocol
+    module Format
+      class Markdown < Base
+        def self.parse(raw, path: nil)
+          raw = raw.dup.force_encoding(Encoding::UTF_8)
+          raise BadFrontmatter.new(path, "entry is not valid UTF-8") unless raw.valid_encoding?
+          return { "_meta" => {}, "body" => raw, "content" => nil } unless raw.start_with?("---\n") || raw.start_with?("---\r\n")
+
+          lines = raw.split(/\r?\n/, -1)
+          close_idx = lines[1..].index("---")
+          raise BadFrontmatter.new(path, "frontmatter not terminated") unless close_idx
+
+          close_idx += 1
+          fm_yaml = lines[1...close_idx].join("\n")
+          body = lines[(close_idx + 1)..].join("\n")
+          begin
+            fm = fm_yaml.strip.empty? ? {} : ::YAML.safe_load(fm_yaml, permitted_classes: [Date, Time], aliases: false)
+          rescue Psych::SyntaxError => e
+            raise BadFrontmatter.new(path, "YAML parse failed: #{e.message}")
+          end
+          fm = {} unless fm.is_a?(Hash)
+          { "_meta" => fm, "body" => body, "content" => nil }
+        end
+
+        def self.serialize(meta:, body:, content: nil)
+          _ = content
+          fm_yaml = meta.empty? ? "" : ::YAML.dump(meta).sub(/\A---\n/, "")
+          body = body.to_s
+          body += "\n" unless body.empty? || body.end_with?("\n")
+          "---\n#{fm_yaml}---\n#{body}"
+        end
+
+        def self.extensions = [".md"]
+        def self.nested_glob = "**/*.md"
+
+        def self.validate_path_extension(path, _nested)
+          ext = File.extname(path)
+          return if ["", ".md"].include?(ext)
+
+          raise UsageError.new("markdown format requires '.md' path (got #{ext.inspect})")
+        end
+
+        def self.data_to_payload(data)
+          data = data.transform_keys(&:to_s) if data.is_a?(Hash)
+          { meta: data["_meta"] || {}, body: (data["body"] || "").to_s, content: nil }
+        end
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/python.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Python < Script
+        def self.extensions = [".py"]
+        def self.nested_ext(*) = ".py"
+      end
+    end
+  end
+end

data/lib/textus/protocol/format/ruby.rb

@@ -0,0 +1,10 @@
+module Textus
+  module Protocol
+    module Format
+      class Ruby < Script
+        def self.extensions = [".rb"]
+        def self.nested_ext(*) = ".rb"
+      end
+    end
+  end
+end