foxtail-tools 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6d238709640d88ba5666a1047d51bac44bf9153ff78bf82c12acfa32ea432bfb
+  data.tar.gz: 35acfed51d9030564151109c61e2d8641e6152c16e5acb169eb3f3eb5894d194
+SHA512:
+  metadata.gz: 0cde2d406e429b21a9684b6c5c68d1061dddb63734ac13de3cc21bea755d8cfbb9fbaf98581c3bfcc4106a72e05b452fc85a44c557ac69d04fca854509e3b022
+  data.tar.gz: 787d507da2c27350b4badf12fa8eabbd75f3e4d626fe99028a234a2bf387b59aa9c8653bff789f66c22d52cc6a853b65ca7f9014b89f8305159da05122ff2ecf

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+## [Unreleased]
+
+## [0.5.0] - 2026-05-08
+
+### Added
+- **Syntax Tooling**
+  - FTL syntax parser with AST support
+  - Serializer for round-trip formatting
+- **CLI Commands**
+  - `foxtail check` - Check FTL files for syntax errors
+  - `foxtail dump` - Dump FTL files as AST in JSON format
+  - `foxtail ids` - Extract message and term IDs from FTL files
+  - `foxtail tidy` - Format FTL files with consistent style
+- **Examples**
+  - AST transformation example in `examples/01_prefix_message_ids.rb`
+
+### Compatibility
+- fluent.js syntax parser: 97/98 (99.0%)
+- Ruby: 3.3+ supported

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 OZAWA Sakuro
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,66 @@
+# 🦊 Foxtail Tools 🌐
+
+Tooling for authoring and validating [Project Fluent](https://projectfluent.org/) files in Ruby.
+
+## Features
+
+- Full syntax parser with spans and comments
+- Serializer for round-trip formatting
+- CLI tools: check, dump, ids, tidy
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "foxtail-tools"
+```
+
+Then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install foxtail-tools
+```
+
+## Usage
+
+### CLI
+
+```bash
+foxtail check messages.ftl
+foxtail dump messages.ftl
+foxtail ids messages.ftl
+foxtail tidy messages.ftl
+```
+
+See [doc/cli.md](doc/cli.md) for the full CLI reference.
+
+### Syntax API
+
+```ruby
+require "foxtail-tools"
+
+parser = Foxtail::Syntax::Parser.new
+ast = parser.parse("hello = Hello")
+```
+
+## Compatibility
+
+The syntax parser passes 97 of 98 [fluent-syntax](https://github.com/projectfluent/fluent.js/tree/main/fluent-syntax) test fixtures (99.0%).
+
+The `leading_dots` fixture is a known mismatch — this test also fails in fluent.js itself.
+
+## Documentation
+
+- [CLI Reference](doc/cli.md)
+- [Architecture](doc/architecture.md)
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/exe/foxtail

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "dry/cli"
+require "foxtail-tools"
+
+begin
+  Dry::CLI.new(Foxtail::CLI).call
+rescue Foxtail::CLI::Error => e
+  warn e.message
+  exit 1
+end

data/lib/foxtail/cli/commands/check.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Foxtail
+  # Command-line interface for Foxtail
+  module CLI
+    module Commands
+      # Check FTL files for syntax errors
+      class Check < Dry::CLI::Command
+        desc "Check FTL files for syntax errors"
+
+        argument :files, type: :array, required: true, desc: "FTL files to check"
+
+        option :quiet, type: :flag, default: false, aliases: ["-q"], desc: "Only show errors, no summary"
+
+        # Execute the check command
+        # @param files [Array<String>] FTL files to check
+        # @param quiet [Boolean] Only show errors, no summary
+        # @return [void]
+        def call(files:, quiet:, **)
+          raise Foxtail::CLI::NoFilesError if files.empty?
+
+          total_errors = 0
+          total_files = 0
+
+          files.each do |file|
+            errors = check_file(file)
+            total_files += 1
+            total_errors += errors.size
+
+            errors.each do |error|
+              err.puts error
+            end
+          end
+
+          unless quiet
+            out.puts
+            out.puts "#{total_files} file(s) checked, #{total_errors} error(s) found"
+          end
+
+          raise Foxtail::CLI::CheckError, total_errors if total_errors > 0
+        end
+
+        private def check_file(path)
+          content = File.read(path)
+          parser = Foxtail::Syntax::Parser.new
+          resource = parser.parse(content)
+
+          resource.body.filter_map {|entry|
+            format_junk_error(path, entry) if entry.is_a?(Foxtail::Syntax::Parser::AST::Junk)
+          }
+        end
+
+        private def format_junk_error(path, junk)
+          first_line = junk.content.lines.first&.chomp || ""
+          "#{path}: syntax error: #{first_line}"
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/cli/commands/dump.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Foxtail
+  # Command-line interface for Foxtail
+  module CLI
+    module Commands
+      # Dump FTL files as AST in JSON format
+      class Dump < Dry::CLI::Command
+        desc "Dump FTL files as AST in JSON format"
+
+        argument :files, type: :array, required: true, desc: "FTL files to dump"
+
+        option :with_spans, type: :flag, default: false, desc: "Include span information in output"
+
+        # Execute the dump command
+        # @param files [Array<String>] FTL files to dump
+        # @param with_spans [Boolean] Include span information
+        # @return [void]
+        def call(files:, with_spans:, **)
+          raise Foxtail::CLI::NoFilesError if files.empty?
+
+          results = files.map {|file| dump_file(Pathname(file), with_spans:) }
+
+          output = files.size == 1 ? results.first : results
+          out.puts JSON.pretty_generate(output)
+        end
+
+        private def dump_file(path, with_spans:)
+          content = path.read
+          parser = Foxtail::Syntax::Parser.new(with_spans:)
+          resource = parser.parse(content)
+
+          {
+            "file" => path.to_s,
+            "ast" => resource.to_h
+          }
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/cli/commands/ids.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Foxtail
+  # Command-line interface for Foxtail
+  module CLI
+    module Commands
+      # Extract message and term IDs from FTL files
+      class Ids < Dry::CLI::Command
+        desc "Extract message and term IDs from FTL files"
+
+        argument :files, type: :array, required: true, desc: "FTL files to extract IDs from"
+
+        option :only_messages, type: :flag, default: false, aliases: ["-m"], desc: "Show only message IDs"
+        option :only_terms, type: :flag, default: false, aliases: ["-t"], desc: "Show only term IDs"
+        option :with_attributes, type: :flag, default: false, aliases: ["-a"], desc: "Include attribute names"
+        option :json, type: :flag, default: false, aliases: ["-j"], desc: "Output as JSON array"
+
+        # Execute the ids command
+        # @param files [Array<String>] FTL files to extract IDs from
+        # @param only_messages [Boolean] Show only message IDs
+        # @param only_terms [Boolean] Show only term IDs
+        # @param with_attributes [Boolean] Include attribute names
+        # @param json [Boolean] Output as JSON array
+        # @return [void]
+        def call(files:, only_messages:, only_terms:, with_attributes:, json:, **)
+          raise Foxtail::CLI::NoFilesError if files.empty?
+
+          ids = []
+
+          files.each do |file|
+            ids.concat(extract_ids(file, only_messages:, only_terms:, with_attributes:))
+          end
+
+          if json
+            out.puts JSON.pretty_generate(ids)
+          else
+            ids.each {|id| out.puts id }
+          end
+        end
+
+        private def extract_ids(path, only_messages:, only_terms:, with_attributes:)
+          ids = []
+          content = File.read(path)
+          parser = Foxtail::Syntax::Parser.new
+          resource = parser.parse(content)
+
+          resource.each_entry do |entry|
+            case entry
+            when Foxtail::Syntax::Parser::AST::Message
+              next if only_terms
+
+              ids << entry.id.name
+              ids.concat(attribute_ids(entry)) if with_attributes
+            when Foxtail::Syntax::Parser::AST::Term
+              next if only_messages
+
+              ids << "-#{entry.id.name}"
+              ids.concat(attribute_ids(entry, prefix: "-")) if with_attributes
+            end
+          end
+
+          ids
+        end
+
+        private def attribute_ids(entry, prefix: "")
+          entry.attributes.map {|attr| "#{prefix}#{entry.id.name}.#{attr.id.name}" }
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/cli/commands/tidy.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Foxtail
+  # Command-line interface for Foxtail
+  module CLI
+    module Commands
+      # Format FTL files with consistent style
+      class Tidy < Dry::CLI::Command
+        desc "Format FTL files with consistent style"
+
+        argument :files, type: :array, required: true, desc: "FTL files to format"
+
+        option :write, type: :flag, default: false, aliases: ["-w"], desc: "Write result back to source file"
+        option :check, type: :flag, default: false, aliases: ["-c"], desc: "Check if files are formatted (for CI)"
+        option :diff, type: :flag, default: false, aliases: ["-d"], desc: "Show diff instead of formatted output"
+        option :with_junk, type: :flag, default: false, desc: "Allow formatting files with syntax errors"
+
+        # Execute the tidy command
+        # @param files [Array<String>] FTL files to format
+        # @param write [Boolean] Write result back to source file
+        # @param check [Boolean] Check if files are formatted (for CI)
+        # @param diff [Boolean] Show diff instead of formatted output
+        # @param with_junk [Boolean] Allow formatting files with syntax errors
+        # @return [void]
+        def call(files:, write:, check:, diff:, with_junk:, **)
+          raise Foxtail::CLI::NoFilesError if files.empty?
+
+          files_with_errors = []
+          files_needing_format = []
+          multiple_files = files.size > 1
+
+          files.each do |file|
+            result = process_file(file, write:, check:, diff:, with_junk:, multiple_files:)
+            case result
+            when :has_errors
+              files_with_errors << file
+            when :needs_format
+              files_needing_format << file
+            end
+          end
+
+          raise Foxtail::CLI::TidyError, files_with_errors unless files_with_errors.empty?
+          raise Foxtail::CLI::TidyCheckError, files_needing_format if check && !files_needing_format.empty?
+        end
+
+        private def process_file(path, write:, check:, diff:, with_junk:, multiple_files:)
+          content = File.read(path)
+          parser = Foxtail::Syntax::Parser.new
+          resource = parser.parse(content)
+
+          # Check for Junk entries (syntax errors)
+          has_junk = resource.body.any?(Foxtail::Syntax::Parser::AST::Junk)
+          return :has_errors if has_junk && !with_junk
+
+          serializer = Foxtail::Syntax::Serializer.new(with_junk:)
+          formatted = serializer.serialize(resource)
+
+          if check
+            return :needs_format if content != formatted
+
+            return :ok
+          end
+
+          if diff
+            output_diff(path, content, formatted)
+            return :ok
+          end
+
+          if write
+            File.write(path, formatted) if content != formatted
+            return :ok
+          end
+
+          # Default: output to stdout
+          output_formatted(path, formatted, multiple_files)
+          :ok
+        end
+
+        private def output_formatted(path, formatted, multiple_files)
+          if multiple_files
+            out.puts "==> #{path} <=="
+            out.puts formatted
+          else
+            out.print formatted
+          end
+        end
+
+        private def output_diff(path, original, formatted)
+          return if original == formatted
+
+          require "tempfile"
+
+          Tempfile.create(["original", ".ftl"]) do |orig_file|
+            Tempfile.create(["formatted", ".ftl"]) do |fmt_file|
+              orig_file.write(original)
+              orig_file.flush
+              fmt_file.write(formatted)
+              fmt_file.flush
+
+              out.print IO.popen(["diff", "-u", "--label", path, orig_file.path, "--label", "#{path} (formatted)", fmt_file.path], &:read)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/cli.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "dry/cli"
+
+module Foxtail
+  # Command-line interface for Foxtail
+  module CLI
+    extend Dry::CLI::Registry
+
+    # Base error class for CLI-specific exceptions
+    class Error < Foxtail::Tools::Error; end
+
+    # Raised when no files are specified for a CLI command
+    class NoFilesError < Error
+      def initialize = super("No files specified")
+    end
+
+    # Raised when check command finds errors in FTL files
+    class CheckError < Error
+      # @return [Integer] Number of errors found
+      attr_reader :error_count
+
+      # @param error_count [Integer] Number of errors found
+      def initialize(error_count)
+        @error_count = error_count
+        super("Check found #{error_count} error(s)")
+      end
+    end
+
+    # Raised when tidy command encounters syntax errors without --with-junk
+    class TidyError < Error
+      # @return [Array<String>] Paths to files with syntax errors
+      attr_reader :files_with_errors
+
+      # @param files_with_errors [Array<String>] Paths to files with syntax errors
+      def initialize(files_with_errors)
+        @files_with_errors = files_with_errors
+        super("Files contain syntax errors: #{files_with_errors.join(", ")}")
+      end
+    end
+
+    # Raised when tidy --check finds files that need formatting
+    class TidyCheckError < Error
+      # @return [Array<String>] Paths to files needing formatting
+      attr_reader :files_needing_format
+
+      # @param files_needing_format [Array<String>] Paths to files needing formatting
+      def initialize(files_needing_format)
+        @files_needing_format = files_needing_format
+        super("Files need formatting: #{files_needing_format.join(", ")}")
+      end
+    end
+
+    register "check", Commands::Check
+    register "dump", Commands::Dump
+    register "ids", Commands::Ids
+    register "tidy", Commands::Tidy
+  end
+end

data/lib/foxtail/syntax/error.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    # Base error class for syntax-specific exceptions
+    class Error < Foxtail::Tools::Error; end
+  end
+end

data/lib/foxtail/syntax/parser/ast/annotation.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Represents parser error annotations with code, arguments, and message
+        class Annotation < SyntaxNode
+          attr_accessor :code
+          attr_accessor :arguments
+          attr_accessor :message
+
+          def initialize(code, arguments=[], message="")
+            super()
+            @code = code
+            @arguments = arguments
+            @message = message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/syntax/parser/ast/attribute.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Represents attributes of messages and terms (e.g., .attr = value)
+        class Attribute < SyntaxNode
+          attr_accessor :id
+          attr_accessor :value
+
+          def initialize(id, value)
+            super()
+            @id = id
+            @value = value
+          end
+
+          def children = [id, value]
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/syntax/parser/ast/base_comment.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Base class for all comment types in Fluent syntax
+        class BaseComment < SyntaxNode
+          attr_accessor :content
+
+          def initialize(content)
+            super()
+            @content = content
+          end
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/syntax/parser/ast/base_literal.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Base class for literal values (strings and numbers) with parsing capability
+        class BaseLiteral < SyntaxNode
+          attr_accessor :value
+
+          def initialize(value)
+            super()
+            # The "value" field contains the exact contents of the literal,
+            # character-for-character.
+            @value = value
+          end
+
+          # Abstract method - subclasses must implement
+          def parse = raise NotImplementedError, "Subclasses must implement parse method"
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/syntax/parser/ast/base_node.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "dry/inflector"
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Base class for all AST nodes in the Fluent parser
+        # Provides common functionality for type management and node equality
+        class BaseNode
+          INFLECTOR = Dry::Inflector.new.freeze
+          private_constant :INFLECTOR
+
+          attr_accessor :type
+
+          def initialize = @type = self.class.name.split("::").last
+
+          # Compare nodes for equality, ignoring span information
+          def ==(other)
+            return false unless other.is_a?(BaseNode)
+
+            this_keys = instance_variables.to_set {|v| v.to_s.delete("@") }
+            other_keys = other.instance_variables.to_set {|v| v.to_s.delete("@") }
+
+            # Always ignore span information in equality comparison
+            this_keys.delete("span")
+            other_keys.delete("span")
+
+            return false if this_keys.size != other_keys.size
+
+            this_keys.each do |field_name|
+              return false unless other_keys.include?(field_name)
+
+              this_val = instance_variable_get("@#{field_name}")
+              other_val = other.instance_variable_get("@#{field_name}")
+
+              return false unless this_val.class == other_val.class
+
+              if this_val.is_a?(Array) && other_val.is_a?(Array)
+                return false if this_val.length != other_val.length
+
+                this_val.each_with_index do |item, i|
+                  return false unless item == other_val[i]
+                end
+              elsif this_val != other_val
+                return false
+              end
+            end
+
+            true
+          end
+
+          # Accept a visitor and dispatch to the appropriate visit method
+          def accept(visitor)
+            method_name = "visit_#{INFLECTOR.underscore(self.class.name.split("::").last)}"
+            visitor.public_send(method_name, self)
+          end
+
+          # Returns an array of child nodes for traversal
+          # Subclasses with children should override this method
+          def children = []
+
+          # Convert node to hash representation for JSON serialization
+          def to_h
+            result = {}
+            instance_variables.each do |var|
+              key = var.to_s.delete("@")
+              value = instance_variable_get(var)
+              result[key] = serialize_value(value)
+            end
+            result
+          end
+
+          private def serialize_value(value)
+            case value
+            when BaseNode
+              value.to_h
+            when Array
+              value.map {|v| serialize_value(v) }
+            else
+              value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/foxtail/syntax/parser/ast/call_arguments.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Foxtail
+  module Syntax
+    class Parser
+      module AST
+        # Represents function call arguments (both positional and named)
+        class CallArguments < SyntaxNode
+          attr_accessor :positional
+          attr_accessor :named
+
+          def initialize(positional=[], named=[])
+            super()
+            @positional = positional
+            @named = named
+          end
+
+          def children = [*positional, *named]
+        end
+      end
+    end
+  end
+end