token-resolver 1.0.0

data/lib/token/resolver/resolve.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Token
+  module Resolver
+    # Resolves tokens in a parsed Document using a replacement map.
+    #
+    # Text nodes pass through unchanged. Token nodes are looked up in the
+    # replacement map by their key. Missing tokens are handled according
+    # to the `on_missing` policy.
+    #
+    # @example Resolve all tokens
+    #   doc = Document.new("Hello {KJ|NAME}!")
+    #   resolver = Resolve.new
+    #   resolver.resolve(doc, {"KJ|NAME" => "World"})
+    #   # => "Hello World!"
+    #
+    # @example Keep unresolved tokens
+    #   resolver = Resolve.new(on_missing: :keep)
+    #   resolver.resolve(doc, {})
+    #   # => "Hello {KJ|NAME}!"
+    #
+    # @example Remove unresolved tokens
+    #   resolver = Resolve.new(on_missing: :remove)
+    #   resolver.resolve(doc, {})
+    #   # => "Hello !"
+    #
+    class Resolve
+      VALID_ON_MISSING = %i[raise keep remove].freeze
+
+      # @return [Symbol] Policy for unresolved tokens (:raise, :keep, :remove)
+      attr_reader :on_missing
+
+      # @param on_missing [Symbol] Behavior for unresolved tokens
+      # @raise [ArgumentError] If on_missing is invalid
+      def initialize(on_missing: :raise)
+        unless VALID_ON_MISSING.include?(on_missing)
+          raise ArgumentError,
+            "Invalid on_missing: #{on_missing.inspect}. Must be one of: #{VALID_ON_MISSING.map(&:inspect).join(", ")}"
+        end
+
+        @on_missing = on_missing
+      end
+
+      # Resolve tokens in a document or node array using a replacement map.
+      #
+      # Resolution is single-pass — replacement values are NOT re-scanned for tokens.
+      #
+      # @param document_or_nodes [Document, Array<Node::Text, Node::Token>] Parsed input
+      # @param replacements [Hash{String => String}] Map of token keys to replacement values
+      # @return [String] Resolved text
+      #
+      # @raise [UnresolvedTokenError] If on_missing is :raise and a token has no replacement
+      def resolve(document_or_nodes, replacements)
+        nodes = case document_or_nodes
+        when Document
+          document_or_nodes.nodes
+        when Array
+          document_or_nodes
+        else
+          raise ArgumentError, "Expected Document or Array of nodes, got #{document_or_nodes.class}"
+        end
+
+        result = +""
+        nodes.each do |node|
+          if node.token?
+            replacement = replacements[node.key]
+            if replacement
+              result << replacement
+            else
+              handle_missing(node, result)
+            end
+          else
+            result << node.to_s
+          end
+        end
+        result
+      end
+
+      private
+
+      def handle_missing(token_node, result)
+        case @on_missing
+        when :raise
+          raise UnresolvedTokenError.new(token_node.key)
+        when :keep
+          result << token_node.to_s
+        when :remove
+          # emit nothing
+        end
+      end
+    end
+  end
+end

data/lib/token/resolver/transform.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require "parslet"
+
+module Token
+  module Resolver
+    # Transforms the raw parslet parse tree into Node::Text and Node::Token objects.
+    #
+    # The grammar produces one `:text` entry per character and `:token` entries
+    # with `:seg` captures. This transform converts those into proper node objects
+    # and coalesces adjacent Text nodes into single nodes.
+    #
+    # @example
+    #   config = Config.default
+    #   tree = Grammar.build(config).new.parse("Hi {KJ|X}!")
+    #   nodes = Transform.apply(tree, config)
+    #   # => [Node::Text("Hi "), Node::Token(["KJ", "X"]), Node::Text("!")]
+    #
+    class Transform
+      class << self
+        # Transform a parslet tree into an array of Text and Token nodes.
+        #
+        # @param tree [Array<Hash>] Raw parslet parse tree
+        # @param config [Config] Token configuration
+        # @return [Array<Node::Text, Node::Token>] Coalesced node array
+        def apply(tree, config)
+          return [] if tree.nil? || (tree.respond_to?(:empty?) && tree.empty?)
+
+          # Convert raw parslet entries to node objects
+          raw_nodes = tree.map { |entry| convert_entry(entry, config) }
+
+          # Coalesce adjacent Text nodes
+          coalesce(raw_nodes)
+        end
+
+        private
+
+        def convert_entry(entry, config)
+          if entry.key?(:token)
+            convert_token(entry[:token], config)
+          elsif entry.key?(:text)
+            Node::Text.new(slice_to_s(entry[:text]))
+          else
+            # Shouldn't happen with our grammar, but be safe
+            Node::Text.new(entry.to_s)
+          end
+        end
+
+        def convert_token(token_data, config)
+          # token_data contains :seg captures
+          # It can be a single seg hash or an array of seg hashes
+          segments = extract_segments(token_data)
+          Node::Token.new(segments, config)
+        end
+
+        def extract_segments(token_data)
+          # Parslet returns different structures depending on repetition:
+          # - Single segment match: {:seg => "value"}
+          # - Multiple segments: [{:seg => "val1"}, {:seg => "val2"}] or {:seg => [...]}
+          case token_data
+          when Hash
+            if token_data[:seg].is_a?(Array)
+              token_data[:seg].map { |s| slice_to_s(s) }
+            else
+              [slice_to_s(token_data[:seg])]
+            end
+          when Array
+            token_data.flat_map { |item|
+              if item.is_a?(Hash) && item.key?(:seg)
+                [slice_to_s(item[:seg])]
+              else
+                [slice_to_s(item)]
+              end
+            }
+          else
+            [slice_to_s(token_data)]
+          end
+        end
+
+        def slice_to_s(value)
+          value.to_s
+        end
+
+        def coalesce(nodes)
+          return nodes if nodes.length <= 1
+
+          nodes.chunk { |node| node.is_a?(Node::Text) }
+            .flat_map { |is_text, group|
+              if is_text
+                combined = group.map(&:content).join
+                [Node::Text.new(combined)]
+              else
+                group
+              end
+            }
+        end
+      end
+    end
+  end
+end

data/lib/token/resolver/version.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Token
+  module Resolver
+    # Version information for Token::Resolver
+    module Version
+      # Current version of the token-resolver gem
+      VERSION = "1.0.0"
+    end
+    VERSION = Version::VERSION # traditional location
+  end
+end

data/lib/token/resolver.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# External gems
+require "version_gem"
+
+# This gem - only version can be required (never autoloaded)
+require_relative "resolver/version"
+
+module Token
+  # Token::Resolver provides configurable PEG-based (parslet) parsing and resolution
+  # of structured tokens in arbitrary text.
+  #
+  # Tokens are configurable structured patterns like `{KJ|GEM_NAME}` that can be
+  # detected in any file format and resolved against a replacement map.
+  #
+  # @example Parse a document to find tokens
+  #   doc = Token::Resolver.parse("Hello {KJ|NAME}!")
+  #   doc.token_keys  # => ["KJ|NAME"]
+  #
+  # @example Resolve tokens in one step
+  #   result = Token::Resolver.resolve(
+  #     "Hello {KJ|NAME}, welcome to {KJ|PROJECT}!",
+  #     {"KJ|NAME" => "World", "KJ|PROJECT" => "token-resolver"}
+  #   )
+  #   # => "Hello World, welcome to token-resolver!"
+  #
+  module Resolver
+    # Base error class for all token-resolver operations.
+    # @api public
+    class Error < StandardError; end
+
+    # Raised when a token has no replacement value and on_missing is :raise.
+    # @api public
+    class UnresolvedTokenError < Error
+      # @return [String] The token key that was not found
+      attr_reader :token_key
+
+      # @param token_key [String] The unresolved token key
+      # @param message [String, nil] Optional custom message
+      def initialize(token_key, message = nil)
+        @token_key = token_key
+        super(message || "Unresolved token: #{token_key}")
+      end
+    end
+
+    # Autoload all classes
+    autoload :Config, "token/resolver/config"
+    autoload :Document, "token/resolver/document"
+    autoload :Grammar, "token/resolver/grammar"
+    autoload :Node, "token/resolver/node"
+    autoload :Resolve, "token/resolver/resolve"
+    autoload :Transform, "token/resolver/transform"
+
+    class << self
+      # Parse input text and return a Document.
+      #
+      # @param input [String] Text to parse for tokens
+      # @param config [Config] Token configuration (default: Config.default)
+      # @return [Document] Parsed document with text and token nodes
+      #
+      # @example
+      #   doc = Token::Resolver.parse("Hello {KJ|NAME}!")
+      #   doc.tokens.first.key  # => "KJ|NAME"
+      def parse(input, config: Config.default)
+        Document.new(input, config: config)
+      end
+
+      # Parse and resolve tokens in one step.
+      #
+      # @param input [String] Text containing tokens to resolve
+      # @param replacements [Hash{String => String}] Map of token keys to replacement values
+      # @param config [Config] Token configuration (default: Config.default)
+      # @param on_missing [Symbol] Behavior for unresolved tokens (:raise, :keep, :remove)
+      # @return [String] Resolved text with tokens replaced
+      #
+      # @example
+      #   Token::Resolver.resolve("{KJ|NAME}", {"KJ|NAME" => "World"})
+      #   # => "World"
+      def resolve(input, replacements, config: Config.default, on_missing: :raise)
+        doc = parse(input, config: config)
+        resolver = Resolve.new(on_missing: on_missing)
+        resolver.resolve(doc, replacements)
+      end
+    end
+  end
+end
+
+Token::Resolver::Version.class_eval do
+  extend VersionGem::Basic
+end

data/lib/token-resolver.rb

@@ -0,0 +1,4 @@
+# For technical reasons, if we move to Zeitwerk, this cannot be require_relative.
+#   See: https://github.com/fxn/zeitwerk#for_gem_extension
+# Hook for other libraries to load this library (e.g. via bundler)
+require "token/resolver"

data/sig/token/resolver.rbs

@@ -0,0 +1,6 @@
+module Token
+  module Resolver
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

data.tar.gz.sig

@@ -0,0 +1,2 @@
+?<"�˽��(���S��wzӈ�_�B�+7�i�H�nxv���ۨ��t"��Y9��u��0Ti��P�]/�Z�03�vh���ڸ�B�
8*�v�@�
+�v-�Y5[�Q�
3fG�)8T{)+��
��C[2������ъ�ߌ�q��a��_��9���,��:�D�6���A����i�5i�@�
�Uv��e�?�����d��$�Bí�%y ���d�2/9�

metadata

@@ -0,0 +1,314 @@
+--- !ruby/object:Gem::Specification
+name: token-resolver
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Peter H. Boling
+bindir: exe
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEgDCCAuigAwIBAgIBATANBgkqhkiG9w0BAQsFADBDMRUwEwYDVQQDDAxwZXRl
+  ci5ib2xpbmcxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixkARkW
+  A2NvbTAeFw0yNTA1MDQxNTMzMDlaFw00NTA0MjkxNTMzMDlaMEMxFTATBgNVBAMM
+  DHBldGVyLmJvbGluZzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYKCZImiZPy
+  LGQBGRYDY29tMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAruUoo0WA
+  uoNuq6puKWYeRYiZekz/nsDeK5x/0IEirzcCEvaHr3Bmz7rjo1I6On3gGKmiZs61
+  LRmQ3oxy77ydmkGTXBjruJB+pQEn7UfLSgQ0xa1/X3kdBZt6RmabFlBxnHkoaGY5
+  mZuZ5+Z7walmv6sFD9ajhzj+oIgwWfnEHkXYTR8I6VLN7MRRKGMPoZ/yvOmxb2DN
+  coEEHWKO9CvgYpW7asIihl/9GMpKiRkcYPm9dGQzZc6uTwom1COfW0+ZOFrDVBuV
+  FMQRPswZcY4Wlq0uEBLPU7hxnCL9nKK6Y9IhdDcz1mY6HZ91WImNslOSI0S8hRpj
+  yGOWxQIhBT3fqCBlRIqFQBudrnD9jSNpSGsFvbEijd5ns7Z9ZMehXkXDycpGAUj1
+  to/5cuTWWw1JqUWrKJYoifnVhtE1o1DZ+LkPtWxHtz5kjDG/zR3MG0Ula0UOavlD
+  qbnbcXPBnwXtTFeZ3C+yrWpE4pGnl3yGkZj9SMTlo9qnTMiPmuWKQDatAgMBAAGj
+  fzB9MAkGA1UdEwQCMAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBQE8uWvNbPVNRXZ
+  HlgPbc2PCzC4bjAhBgNVHREEGjAYgRZwZXRlci5ib2xpbmdAZ21haWwuY29tMCEG
+  A1UdEgQaMBiBFnBldGVyLmJvbGluZ0BnbWFpbC5jb20wDQYJKoZIhvcNAQELBQAD
+  ggGBAJbnUwfJQFPkBgH9cL7hoBfRtmWiCvdqdjeTmi04u8zVNCUox0A4gT982DE9
+  wmuN12LpdajxZONqbXuzZvc+nb0StFwmFYZG6iDwaf4BPywm2e/Vmq0YG45vZXGR
+  L8yMDSK1cQXjmA+ZBKOHKWavxP6Vp7lWvjAhz8RFwqF9GuNIdhv9NpnCAWcMZtpm
+  GUPyIWw/Cw/2wZp74QzZj6Npx+LdXoLTF1HMSJXZ7/pkxLCsB8m4EFVdb/IrW/0k
+  kNSfjtAfBHO8nLGuqQZVH9IBD1i9K6aSs7pT6TW8itXUIlkIUI2tg5YzW6OFfPzq
+  QekSkX3lZfY+HTSp/o+YvKkqWLUV7PQ7xh1ZYDtocpaHwgxe/j3bBqHE+CUPH2vA
+  0V/FwdTRWcwsjVoOJTrYcff8pBZ8r2MvtAc54xfnnhGFzeRHfcltobgFxkAXdE6p
+  DVjBtqT23eugOqQ73umLcYDZkc36vnqGxUBSsXrzY9pzV5gGr2I8YUxMqf6ATrZt
+  L9nRqA==
+  -----END CERTIFICATE-----
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: version_gem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.9
+- !ruby/object:Gem::Dependency
+  name: kettle-dev
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: bundler-audit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: require_bench
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+- !ruby/object:Gem::Dependency
+  name: appraisal2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+- !ruby/object:Gem::Dependency
+  name: kettle-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.10
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.10
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: stone_checksums
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: gitmoji-regex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+description: "\U0001FA99 Token::Resolver provides configurable PEG-based (parslet)
+  parsing and resolution of structured tokens (e.g., {KJ|GEM_NAME}) in arbitrary text.
+  Useful for template ETL pipelines where tokens in template files must be resolved
+  before format-specific merging."
+email:
+- floss@galtzo.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- CHANGELOG.md
+- CITATION.cff
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- FUNDING.md
+- LICENSE.txt
+- README.md
+- REEK
+- RUBOCOP.md
+- SECURITY.md
+files:
+- CHANGELOG.md
+- CITATION.cff
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- FUNDING.md
+- LICENSE.txt
+- README.md
+- REEK
+- RUBOCOP.md
+- SECURITY.md
+- lib/token-resolver.rb
+- lib/token/resolver.rb
+- lib/token/resolver/config.rb
+- lib/token/resolver/document.rb
+- lib/token/resolver/grammar.rb
+- lib/token/resolver/node.rb
+- lib/token/resolver/node/text.rb
+- lib/token/resolver/node/token.rb
+- lib/token/resolver/resolve.rb
+- lib/token/resolver/transform.rb
+- lib/token/resolver/version.rb
+- sig/token/resolver.rbs
+homepage: https://github.com/kettle-rb/token-resolver
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://token-resolver.galtzo.com/
+  source_code_uri: https://github.com/kettle-rb/token-resolver/tree/v1.0.0
+  changelog_uri: https://github.com/kettle-rb/token-resolver/blob/v1.0.0/CHANGELOG.md
+  bug_tracker_uri: https://github.com/kettle-rb/token-resolver/issues
+  documentation_uri: https://www.rubydoc.info/gems/token-resolver/1.0.0
+  funding_uri: https://github.com/sponsors/pboling
+  wiki_uri: https://github.com/kettle-rb/token-resolver/wiki
+  news_uri: https://www.railsbling.com/tags/token-resolver
+  discord_uri: https://discord.gg/3qme4XHNKN
+  rubygems_mfa_required: 'true'
+rdoc_options:
+- "--title"
+- "token-resolver - \U0001FA99 Configurable PEG-based token parser and resolver for
+  structured token detection and replacement in arbitrary text"
+- "--main"
+- README.md
+- "--exclude"
+- "^sig/"
+- "--line-numbers"
+- "--inline-source"
+- "--quiet"
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.5
+specification_version: 4
+summary: "\U0001FA99 Configurable PEG-based token parser and resolver for structured
+  token detection and replacement in arbitrary text"
+test_files: []