ruby_grammar_builder 0.0.1

data/lib/textmate_grammar/transforms/fix_repeated_tag_as.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+#
+# tag_as: inside a quantifier does not work as expected, this fixes it
+#
+class FixRepeatedTagAs < GrammarTransform
+    #
+    # Does pattern or any of its children / siblings have a tag_as
+    #
+    # @param [PatternBase, String] pattern the pattern to check
+    #
+    # @return [Boolean] if any of the patterns have a tag_as
+    #
+    def tag_as?(pattern)
+        return false unless pattern.is_a? PatternBase
+
+        pattern.each do |s|
+            return true if s.arguments[:tag_as]
+        end
+
+        false
+    end
+
+    #
+    # fixes tag_as when it is inside a quantifier
+    # see https://github.com/jeff-hykin/cpp-textmate-grammar/issues/339#issuecomment-543285390
+    # for an explanation of why and how
+    #
+    def pre_transform(pattern, options)
+        return pattern.map { |v| pre_transform(v, options) } if pattern.is_a? Array
+        return pattern unless pattern.is_a? PatternBase
+        return pattern if pattern.is_a? PatternRange
+
+        pattern.map do |pat|
+            next pat unless pat.respond_to? :self_capture_group_rematch
+            next pat unless pat.self_capture_group_rematch
+            next pat unless tag_as?(pat.match)
+
+            unless pat.arguments[:includes].nil? || pat.arguments[:includes].empty?
+                raise "Cannot transform a Repeated pattern that has non empty includes"
+            end
+
+            pat.arguments[:includes] = [pat.match.__deep_clone__]
+            pat.match.map! do |pm|
+                pm.arguments.delete(:tag_as)
+                pm.arguments.delete(:includes)
+                next unless options[:preserve_references?] || pm.arguments[:preserve_references?]
+
+                pm.self_scramble_references
+            end
+        end
+    end
+
+    #
+    # Contributes the option :preserve_references?
+    #
+    # :preserve_references? disables the scrambling of references
+    #
+    # @return (see GrammarPlugin.options)
+    #
+    def self.options
+        [:preserve_references?]
+    end
+
+    #
+    # Displays the state of the options
+    #
+    # @return (see GrammarPlugin.display_options)
+    #
+    def self.display_options(indent, options)
+        ",\n#{indent}preserve_references?: #{options[:preserve_references?]}"
+    end
+end
+
+Grammar.register_transform(FixRepeatedTagAs.new, 99)

data/lib/textmate_grammar/transforms/resolve_placeholders.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+#
+# Resolves any embedded placeholders
+#
+class ResolvePlaceholders < GrammarTransform
+    def pre_transform(pattern, options)
+        return pattern unless pattern.is_a? PatternBase
+        pattern_copy = pattern.__deep_clone__
+        pattern_copy.map!(true) do |each_pattern_like|
+            
+            arguments = each_pattern_like.arguments
+            repository = options[:repository]
+            
+            if each_pattern_like.is_a?(PlaceholderPattern)
+                
+                unless repository[arguments[:placeholder]].is_a? PatternBase
+                    raise ":#{arguments[:placeholder]} is not a pattern and cannot be substituted"
+                end
+
+                each_pattern_like.match = repository[arguments[:placeholder]].__deep_clone__
+            #
+            # token pattern
+            #
+            elsif each_pattern_like.is_a?(TokenPattern)
+                qualifying_patterns = []
+                for each_key, each_value in repository
+                    next unless each_value.is_a?(PatternBase)
+                    qualifying_patterns << each_value if arguments[:pattern_filter][each_value]
+                end
+                if qualifying_patterns.size == 0
+                    raise <<-HEREDOC.remove_indent
+                        
+                        
+                        When creating a token filter #{arguments[:pattern_filter]}
+                        all the patterns that are in the grammar repository were searched
+                        but none of thier adjective lists matched the token filter
+                    HEREDOC
+                end
+                
+                
+                # change this pattern right before the grammar is generated
+                each_pattern_like.match = oneOf(qualifying_patterns)
+            end
+            each_pattern_like
+        end
+        pattern_copy.freeze
+    end
+end
+                        
+# resolving placeholders has no dependencies and makes analyzing patterns much nicer
+# so it happens fairly early
+Grammar.register_transform(ResolvePlaceholders.new, 0)
+
+
+
+
+
+# # frozen_string_literal: true
+
+# #
+# # Resolves any embedded placeholders
+# #
+# class ResolvePlaceholders < GrammarTransform
+#     def pre_transform(pattern, options)
+#         # skip past anything that isn't a pattern
+#         return pattern unless pattern.is_a? PatternBase
+        
+#         pattern_copy = pattern.__deep_clone__
+#         # recursively fill in all of the placeholders by looking them up
+#         repository = options[:repository]
+#         pattern_copy.map!(true) do |each_pattern_like|
+#             arguments = each_pattern_like.arguments
+#             name_of_placeholder = arguments[:placeholder]
+#             #
+#             # placeholder pattern
+#             #
+#             if each_pattern_like.is_a?(PlaceholderPattern)
+#                 # error if can't find thing the placeholder is reffering to 
+#                 if !repository[name_of_placeholder].is_a?(PatternBase)
+#                     raise "\n#{arguments[:placeholder]} is not a pattern and cannot be substituted"
+#                 end
+                
+#                 # if the pattern exists though, make the substitution
+#                 arguments = { match:repository[arguments[:placeholder]].__deep_clone__ }
+#                 for each_key, each_value in each_pattern_like.arguments
+#                     arguments[each_key]  = each_value
+#                 end
+#                 each_pattern_like = Pattern.new(arguments)
+#             #
+#             # token pattern
+#             #
+#             elsif each_pattern_like.is_a?(TokenPattern)
+#                 qualifying_patterns = []
+#                 for each_key, each_value in repository
+#                     next unless each_value.is_a?(PatternBase)
+#                     qualifying_patterns << each_value if arguments[:pattern_filter][each_value]
+#                 end
+#                 if qualifying_patterns.size == 0
+#                     raise <<-HEREDOC.remove_indent
+                        
+                        
+#                         When creating a token filter #{arguments[:pattern_filter]}
+#                         all the patterns that are in the grammar repository were searched
+#                         but none of thier adjective lists matched the token filter
+#                     HEREDOC
+#                 end
+                
+                
+#                 # change this pattern right before the grammar is generated
+#                 each_pattern_like.match = oneOf(qualifying_patterns)
+#             end
+#             each_pattern_like
+#         end
+#         return pattern_copy
+#     end
+# end
+                        
+# # resolving placeholders has no dependencies and makes analyzing patterns much nicer
+# # so it happens fairly early
+# Grammar.register_transform(ResolvePlaceholders.new, 0)

data/lib/textmate_grammar/util.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+#
+# Disables warnings for the block
+#
+# @return [void]
+#
+def with_no_warnings
+    old_verbose = $VERBOSE
+    $VERBOSE = nil
+    yield
+ensure
+    $VERBOSE = old_verbose
+end
+
+# Add remove indent to the String class
+class String
+    # a helper for writing multi-line strings for error messages
+    # example usage
+    #     puts <<-HEREDOC.remove_indent
+    #     This command does such and such.
+    #         this part is extra indented
+    #     HEREDOC
+    # @return [String]
+    def remove_indent
+        gsub(/^[ \t]{#{match(/^[ \t]*/)[0].length}}/, '')
+    end
+end
+
+#
+# Provides to_s
+#
+class Enumerator
+    #
+    # Converts Enumerator to a string representing Integer.times
+    #
+    # @return [String] the Enumerator as a string
+    #
+    def to_s
+        size.to_s + ".times"
+    end
+end
+
+# determines if a regex string is a single entity
+#
+# @note single entity means that for the purposes of modification, the expression is
+#   atomic, for example if appending a +*+ to the end of +regex_string+ matches only
+#   a part of regex string multiple times then it is not a single_entity
+# @param regex_string [String] a string representing a regular expression, without the
+#   forward slash "/" at the beginning and
+# @return [Boolean] if the string represents an single regex entity
+def string_single_entity?(regex_string)
+    escaped = false
+    in_set = false
+    depth = 0
+    regex_string.each_char.with_index do |c, index|
+        # allow the first character to be at depth 0
+        # NOTE: this automatically makes a single char regexp a single entity
+        return false if depth == 0 && index != 0
+
+        if escaped
+            escaped = false
+            next
+        end
+        if c == '\\'
+            escaped = true
+            next
+        end
+        if in_set
+            if c == ']'
+                in_set = false
+                depth -= 1
+            end
+            next
+        end
+        case c
+        when "(" then depth += 1
+        when ")" then depth -= 1
+        when "["
+            depth += 1
+            in_set = true
+        end
+    end
+    # sanity check
+    if depth != 0 or escaped or in_set
+        puts "Internal error: when determining if a Regexp is a single entity"
+        puts "an unexpected sequence was found. This is a bug with the gem."
+        puts "This will not effect the validity of the produced grammar"
+        puts "Regexp: #{inspect} depth: #{depth} escaped: #{escaped} in_set: #{in_set}"
+        return false
+    end
+    true
+end
+
+#
+# Wraps a pattern in start and end anchors
+#
+# @param [PatternBase] pat the pattern to wrap
+#
+# @return [PatternBase] the wrapped pattern
+#
+def wrap_with_anchors(pat)
+    Pattern.new(/^/).then(Pattern.new(pat)).then(/$/)
+end
+
+#
+# Fixes value to be either a PatternBase, Symbol, or Array of either
+#
+# @param [*] value The value to fixup
+#
+# @return [PatternBase,Symbol,Array<PatternBase,Symbol>] the fixed value
+#
+def fixup_value(value)
+    is_array = value.is_a? Array
+    # ensure array is flat and only contains patterns or symbols
+    value = [value].flatten.map do |v|
+        next v if v.is_a? Symbol
+
+        if v.is_a? String
+            next v if v.start_with?("source.", "text.", "$")
+        end
+
+        if v.is_a? Hash
+            # check for an implicit legacy pattern
+            legacy_keys = [
+                :name,
+                :contentName,
+                :begin,
+                :end,
+                :while,
+                :comment,
+                :disabled,
+                :patterns,
+            ]
+            v = LegacyPattern.new(v) unless (v.keys.map(&:to_sym) & legacy_keys).empty?
+        end
+
+        v = Pattern.new(v) unless v.is_a? PatternBase
+        v
+    end
+
+    value = value[0] unless is_array
+    value
+end
+
+#
+# Determine the absolute path that a require statement resolves to
+#
+# @note this assumes path was successfully required previously
+#
+# @param [String] path the path to resolve
+#
+# @return [String] the resolved path
+#
+def resolve_require(path)
+    path = Pathname.new path
+    return path.to_s if path.absolute? && path.extname != ""
+
+    return path.dirname.glob("#{path.basename}.{rb,so,dll}")[0].to_s if path.absolute?
+
+    $LOAD_PATH.each do |p|
+        test_path = Pathname.new(p).join(path)
+        return test_path.to_s if path.extname != "" && test_path.exist?
+
+        test_paths = test_path.dirname.glob("#{test_path.basename}.{rb,so,dll}")
+        return test_paths[0].to_s unless test_paths.empty?
+    end
+
+    ""
+end
+
+#
+# Converts an output into a set of tags
+#
+# @param [Hash] output output of Grammar#generate
+#
+# @return [Set<String>] The tags in the grammar
+#
+def get_tags(output)
+    repository = output[:repository]
+    repository[:$initial_context] = {patterns: output[:patterns]}
+    tags = Set.new
+    add_tags = lambda do |rule|
+        rule = rule.transform_keys(&:to_sym)
+        tags.merge(rule[:name].split(" ")) if rule[:name]
+        tags.merge(rule[:contentName].split(" ")) if rule[:contentName]
+
+        rule[:patterns]&.each { |p| add_tags.call(p) }
+        rule[:captures]&.values&.each { |p| add_tags.call(p) }
+        rule[:beginCaptures]&.values&.each { |p| add_tags.call(p) }
+        rule[:endCaptures]&.values&.each { |p| add_tags.call(p) }
+        rule[:whileCaptures]&.values&.each { |p| add_tags.call(p) }
+    end
+
+    repository.values.each { |p| add_tags.call(p) }
+
+    tags
+end

data/lib/textmate_grammar.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'ruby_grammar_builder/grammar'
+require 'ruby_grammar_builder/stdlib/common'

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification
+name: ruby_grammar_builder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jeff Hykin
+- Matthew Fosdick
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-09-16 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- lib/textmate_grammar.rb
+- lib/textmate_grammar/generated/grammar.rb
+- lib/textmate_grammar/generated/rule.rb
+- lib/textmate_grammar/grammar.rb
+- lib/textmate_grammar/grammar_plugin.rb
+- lib/textmate_grammar/import_patterns.rb
+- lib/textmate_grammar/linters/flat_includes.rb
+- lib/textmate_grammar/linters/includes_then_tag_as.rb
+- lib/textmate_grammar/linters/standard_naming.rb
+- lib/textmate_grammar/linters/start_match_empty.rb
+- lib/textmate_grammar/linters/tests.rb
+- lib/textmate_grammar/linters/unused_unresolved.rb
+- lib/textmate_grammar/pattern_extensions/look_ahead_for.rb
+- lib/textmate_grammar/pattern_extensions/look_ahead_to_avoid.rb
+- lib/textmate_grammar/pattern_extensions/look_behind_for.rb
+- lib/textmate_grammar/pattern_extensions/look_behind_to_avoid.rb
+- lib/textmate_grammar/pattern_extensions/lookaround_pattern.rb
+- lib/textmate_grammar/pattern_extensions/match_result_of.rb
+- lib/textmate_grammar/pattern_extensions/maybe.rb
+- lib/textmate_grammar/pattern_extensions/one_of.rb
+- lib/textmate_grammar/pattern_extensions/one_or_more_of.rb
+- lib/textmate_grammar/pattern_extensions/or_pattern.rb
+- lib/textmate_grammar/pattern_extensions/placeholder.rb
+- lib/textmate_grammar/pattern_extensions/recursively_match.rb
+- lib/textmate_grammar/pattern_extensions/zero_or_more_of.rb
+- lib/textmate_grammar/pattern_variations/base_pattern.rb
+- lib/textmate_grammar/pattern_variations/legacy_pattern.rb
+- lib/textmate_grammar/pattern_variations/pattern.rb
+- lib/textmate_grammar/pattern_variations/pattern_range.rb
+- lib/textmate_grammar/pattern_variations/repeatable_pattern.rb
+- lib/textmate_grammar/regex_operator.rb
+- lib/textmate_grammar/regex_operators/alternation.rb
+- lib/textmate_grammar/regex_operators/concat.rb
+- lib/textmate_grammar/stdlib/common.rb
+- lib/textmate_grammar/tokens.rb
+- lib/textmate_grammar/transforms/add_ending.rb
+- lib/textmate_grammar/transforms/bailout.rb
+- lib/textmate_grammar/transforms/fix_repeated_tag_as.rb
+- lib/textmate_grammar/transforms/resolve_placeholders.rb
+- lib/textmate_grammar/util.rb
+homepage: https://github.com/jeff-hykin/cpp-textmate-grammar
+licenses:
+- MIT
+metadata:
+  yard.run: yri
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.16
+signing_key:
+specification_version: 4
+summary: A library to generate textmate grammars
+test_files: []