prism 0.13.0 → 0.14.0

data/lib/prism/compiler.rb

@@ -39,426 +39,426 @@ module Prism
       node.compact_child_nodes.map { |node| node.accept(self) }
     end
 
-        # Compile a AliasGlobalVariableNode node
+    # Compile a AliasGlobalVariableNode node
     alias visit_alias_global_variable_node visit_child_nodes
-    
+
     # Compile a AliasMethodNode node
     alias visit_alias_method_node visit_child_nodes
-    
+
     # Compile a AlternationPatternNode node
     alias visit_alternation_pattern_node visit_child_nodes
-    
+
     # Compile a AndNode node
     alias visit_and_node visit_child_nodes
-    
+
     # Compile a ArgumentsNode node
     alias visit_arguments_node visit_child_nodes
-    
+
     # Compile a ArrayNode node
     alias visit_array_node visit_child_nodes
-    
+
     # Compile a ArrayPatternNode node
     alias visit_array_pattern_node visit_child_nodes
-    
+
     # Compile a AssocNode node
     alias visit_assoc_node visit_child_nodes
-    
+
     # Compile a AssocSplatNode node
     alias visit_assoc_splat_node visit_child_nodes
-    
+
     # Compile a BackReferenceReadNode node
     alias visit_back_reference_read_node visit_child_nodes
-    
+
     # Compile a BeginNode node
     alias visit_begin_node visit_child_nodes
-    
+
     # Compile a BlockArgumentNode node
     alias visit_block_argument_node visit_child_nodes
-    
+
     # Compile a BlockLocalVariableNode node
     alias visit_block_local_variable_node visit_child_nodes
-    
+
     # Compile a BlockNode node
     alias visit_block_node visit_child_nodes
-    
+
     # Compile a BlockParameterNode node
     alias visit_block_parameter_node visit_child_nodes
-    
+
     # Compile a BlockParametersNode node
     alias visit_block_parameters_node visit_child_nodes
-    
+
     # Compile a BreakNode node
     alias visit_break_node visit_child_nodes
-    
+
     # Compile a CallAndWriteNode node
     alias visit_call_and_write_node visit_child_nodes
-    
+
     # Compile a CallNode node
     alias visit_call_node visit_child_nodes
-    
+
     # Compile a CallOperatorWriteNode node
     alias visit_call_operator_write_node visit_child_nodes
-    
+
     # Compile a CallOrWriteNode node
     alias visit_call_or_write_node visit_child_nodes
-    
+
     # Compile a CapturePatternNode node
     alias visit_capture_pattern_node visit_child_nodes
-    
+
     # Compile a CaseNode node
     alias visit_case_node visit_child_nodes
-    
+
     # Compile a ClassNode node
     alias visit_class_node visit_child_nodes
-    
+
     # Compile a ClassVariableAndWriteNode node
     alias visit_class_variable_and_write_node visit_child_nodes
-    
+
     # Compile a ClassVariableOperatorWriteNode node
     alias visit_class_variable_operator_write_node visit_child_nodes
-    
+
     # Compile a ClassVariableOrWriteNode node
     alias visit_class_variable_or_write_node visit_child_nodes
-    
+
     # Compile a ClassVariableReadNode node
     alias visit_class_variable_read_node visit_child_nodes
-    
+
     # Compile a ClassVariableTargetNode node
     alias visit_class_variable_target_node visit_child_nodes
-    
+
     # Compile a ClassVariableWriteNode node
     alias visit_class_variable_write_node visit_child_nodes
-    
+
     # Compile a ConstantAndWriteNode node
     alias visit_constant_and_write_node visit_child_nodes
-    
+
     # Compile a ConstantOperatorWriteNode node
     alias visit_constant_operator_write_node visit_child_nodes
-    
+
     # Compile a ConstantOrWriteNode node
     alias visit_constant_or_write_node visit_child_nodes
-    
+
     # Compile a ConstantPathAndWriteNode node
     alias visit_constant_path_and_write_node visit_child_nodes
-    
+
     # Compile a ConstantPathNode node
     alias visit_constant_path_node visit_child_nodes
-    
+
     # Compile a ConstantPathOperatorWriteNode node
     alias visit_constant_path_operator_write_node visit_child_nodes
-    
+
     # Compile a ConstantPathOrWriteNode node
     alias visit_constant_path_or_write_node visit_child_nodes
-    
+
     # Compile a ConstantPathTargetNode node
     alias visit_constant_path_target_node visit_child_nodes
-    
+
     # Compile a ConstantPathWriteNode node
     alias visit_constant_path_write_node visit_child_nodes
-    
+
     # Compile a ConstantReadNode node
     alias visit_constant_read_node visit_child_nodes
-    
+
     # Compile a ConstantTargetNode node
     alias visit_constant_target_node visit_child_nodes
-    
+
     # Compile a ConstantWriteNode node
     alias visit_constant_write_node visit_child_nodes
-    
+
     # Compile a DefNode node
     alias visit_def_node visit_child_nodes
-    
+
     # Compile a DefinedNode node
     alias visit_defined_node visit_child_nodes
-    
+
     # Compile a ElseNode node
     alias visit_else_node visit_child_nodes
-    
+
     # Compile a EmbeddedStatementsNode node
     alias visit_embedded_statements_node visit_child_nodes
-    
+
     # Compile a EmbeddedVariableNode node
     alias visit_embedded_variable_node visit_child_nodes
-    
+
     # Compile a EnsureNode node
     alias visit_ensure_node visit_child_nodes
-    
+
     # Compile a FalseNode node
     alias visit_false_node visit_child_nodes
-    
+
     # Compile a FindPatternNode node
     alias visit_find_pattern_node visit_child_nodes
-    
+
     # Compile a FlipFlopNode node
     alias visit_flip_flop_node visit_child_nodes
-    
+
     # Compile a FloatNode node
     alias visit_float_node visit_child_nodes
-    
+
     # Compile a ForNode node
     alias visit_for_node visit_child_nodes
-    
+
     # Compile a ForwardingArgumentsNode node
     alias visit_forwarding_arguments_node visit_child_nodes
-    
+
     # Compile a ForwardingParameterNode node
     alias visit_forwarding_parameter_node visit_child_nodes
-    
+
     # Compile a ForwardingSuperNode node
     alias visit_forwarding_super_node visit_child_nodes
-    
+
     # Compile a GlobalVariableAndWriteNode node
     alias visit_global_variable_and_write_node visit_child_nodes
-    
+
     # Compile a GlobalVariableOperatorWriteNode node
     alias visit_global_variable_operator_write_node visit_child_nodes
-    
+
     # Compile a GlobalVariableOrWriteNode node
     alias visit_global_variable_or_write_node visit_child_nodes
-    
+
     # Compile a GlobalVariableReadNode node
     alias visit_global_variable_read_node visit_child_nodes
-    
+
     # Compile a GlobalVariableTargetNode node
     alias visit_global_variable_target_node visit_child_nodes
-    
+
     # Compile a GlobalVariableWriteNode node
     alias visit_global_variable_write_node visit_child_nodes
-    
+
     # Compile a HashNode node
     alias visit_hash_node visit_child_nodes
-    
+
     # Compile a HashPatternNode node
     alias visit_hash_pattern_node visit_child_nodes
-    
+
     # Compile a IfNode node
     alias visit_if_node visit_child_nodes
-    
+
     # Compile a ImaginaryNode node
     alias visit_imaginary_node visit_child_nodes
-    
+
     # Compile a ImplicitNode node
     alias visit_implicit_node visit_child_nodes
-    
+
     # Compile a InNode node
     alias visit_in_node visit_child_nodes
-    
+
     # Compile a InstanceVariableAndWriteNode node
     alias visit_instance_variable_and_write_node visit_child_nodes
-    
+
     # Compile a InstanceVariableOperatorWriteNode node
     alias visit_instance_variable_operator_write_node visit_child_nodes
-    
+
     # Compile a InstanceVariableOrWriteNode node
     alias visit_instance_variable_or_write_node visit_child_nodes
-    
+
     # Compile a InstanceVariableReadNode node
     alias visit_instance_variable_read_node visit_child_nodes
-    
+
     # Compile a InstanceVariableTargetNode node
     alias visit_instance_variable_target_node visit_child_nodes
-    
+
     # Compile a InstanceVariableWriteNode node
     alias visit_instance_variable_write_node visit_child_nodes
-    
+
     # Compile a IntegerNode node
     alias visit_integer_node visit_child_nodes
-    
+
     # Compile a InterpolatedMatchLastLineNode node
     alias visit_interpolated_match_last_line_node visit_child_nodes
-    
+
     # Compile a InterpolatedRegularExpressionNode node
     alias visit_interpolated_regular_expression_node visit_child_nodes
-    
+
     # Compile a InterpolatedStringNode node
     alias visit_interpolated_string_node visit_child_nodes
-    
+
     # Compile a InterpolatedSymbolNode node
     alias visit_interpolated_symbol_node visit_child_nodes
-    
+
     # Compile a InterpolatedXStringNode node
     alias visit_interpolated_x_string_node visit_child_nodes
-    
+
     # Compile a KeywordHashNode node
     alias visit_keyword_hash_node visit_child_nodes
-    
+
     # Compile a KeywordParameterNode node
     alias visit_keyword_parameter_node visit_child_nodes
-    
+
     # Compile a KeywordRestParameterNode node
     alias visit_keyword_rest_parameter_node visit_child_nodes
-    
+
     # Compile a LambdaNode node
     alias visit_lambda_node visit_child_nodes
-    
+
     # Compile a LocalVariableAndWriteNode node
     alias visit_local_variable_and_write_node visit_child_nodes
-    
+
     # Compile a LocalVariableOperatorWriteNode node
     alias visit_local_variable_operator_write_node visit_child_nodes
-    
+
     # Compile a LocalVariableOrWriteNode node
     alias visit_local_variable_or_write_node visit_child_nodes
-    
+
     # Compile a LocalVariableReadNode node
     alias visit_local_variable_read_node visit_child_nodes
-    
+
     # Compile a LocalVariableTargetNode node
     alias visit_local_variable_target_node visit_child_nodes
-    
+
     # Compile a LocalVariableWriteNode node
     alias visit_local_variable_write_node visit_child_nodes
-    
+
     # Compile a MatchLastLineNode node
     alias visit_match_last_line_node visit_child_nodes
-    
+
     # Compile a MatchPredicateNode node
     alias visit_match_predicate_node visit_child_nodes
-    
+
     # Compile a MatchRequiredNode node
     alias visit_match_required_node visit_child_nodes
-    
+
     # Compile a MatchWriteNode node
     alias visit_match_write_node visit_child_nodes
-    
+
     # Compile a MissingNode node
     alias visit_missing_node visit_child_nodes
-    
+
     # Compile a ModuleNode node
     alias visit_module_node visit_child_nodes
-    
+
     # Compile a MultiTargetNode node
     alias visit_multi_target_node visit_child_nodes
-    
+
     # Compile a MultiWriteNode node
     alias visit_multi_write_node visit_child_nodes
-    
+
     # Compile a NextNode node
     alias visit_next_node visit_child_nodes
-    
+
     # Compile a NilNode node
     alias visit_nil_node visit_child_nodes
-    
+
     # Compile a NoKeywordsParameterNode node
     alias visit_no_keywords_parameter_node visit_child_nodes
-    
+
     # Compile a NumberedReferenceReadNode node
     alias visit_numbered_reference_read_node visit_child_nodes
-    
+
     # Compile a OptionalParameterNode node
     alias visit_optional_parameter_node visit_child_nodes
-    
+
     # Compile a OrNode node
     alias visit_or_node visit_child_nodes
-    
+
     # Compile a ParametersNode node
     alias visit_parameters_node visit_child_nodes
-    
+
     # Compile a ParenthesesNode node
     alias visit_parentheses_node visit_child_nodes
-    
+
     # Compile a PinnedExpressionNode node
     alias visit_pinned_expression_node visit_child_nodes
-    
+
     # Compile a PinnedVariableNode node
     alias visit_pinned_variable_node visit_child_nodes
-    
+
     # Compile a PostExecutionNode node
     alias visit_post_execution_node visit_child_nodes
-    
+
     # Compile a PreExecutionNode node
     alias visit_pre_execution_node visit_child_nodes
-    
+
     # Compile a ProgramNode node
     alias visit_program_node visit_child_nodes
-    
+
     # Compile a RangeNode node
     alias visit_range_node visit_child_nodes
-    
+
     # Compile a RationalNode node
     alias visit_rational_node visit_child_nodes
-    
+
     # Compile a RedoNode node
     alias visit_redo_node visit_child_nodes
-    
+
     # Compile a RegularExpressionNode node
     alias visit_regular_expression_node visit_child_nodes
-    
+
     # Compile a RequiredDestructuredParameterNode node
     alias visit_required_destructured_parameter_node visit_child_nodes
-    
+
     # Compile a RequiredParameterNode node
     alias visit_required_parameter_node visit_child_nodes
-    
+
     # Compile a RescueModifierNode node
     alias visit_rescue_modifier_node visit_child_nodes
-    
+
     # Compile a RescueNode node
     alias visit_rescue_node visit_child_nodes
-    
+
     # Compile a RestParameterNode node
     alias visit_rest_parameter_node visit_child_nodes
-    
+
     # Compile a RetryNode node
     alias visit_retry_node visit_child_nodes
-    
+
     # Compile a ReturnNode node
     alias visit_return_node visit_child_nodes
-    
+
     # Compile a SelfNode node
     alias visit_self_node visit_child_nodes
-    
+
     # Compile a SingletonClassNode node
     alias visit_singleton_class_node visit_child_nodes
-    
+
     # Compile a SourceEncodingNode node
     alias visit_source_encoding_node visit_child_nodes
-    
+
     # Compile a SourceFileNode node
     alias visit_source_file_node visit_child_nodes
-    
+
     # Compile a SourceLineNode node
     alias visit_source_line_node visit_child_nodes
-    
+
     # Compile a SplatNode node
     alias visit_splat_node visit_child_nodes
-    
+
     # Compile a StatementsNode node
     alias visit_statements_node visit_child_nodes
-    
+
     # Compile a StringConcatNode node
     alias visit_string_concat_node visit_child_nodes
-    
+
     # Compile a StringNode node
     alias visit_string_node visit_child_nodes
-    
+
     # Compile a SuperNode node
     alias visit_super_node visit_child_nodes
-    
+
     # Compile a SymbolNode node
     alias visit_symbol_node visit_child_nodes
-    
+
     # Compile a TrueNode node
     alias visit_true_node visit_child_nodes
-    
+
     # Compile a UndefNode node
     alias visit_undef_node visit_child_nodes
-    
+
     # Compile a UnlessNode node
     alias visit_unless_node visit_child_nodes
-    
+
     # Compile a UntilNode node
     alias visit_until_node visit_child_nodes
-    
+
     # Compile a WhenNode node
     alias visit_when_node visit_child_nodes
-    
+
     # Compile a WhileNode node
     alias visit_while_node visit_child_nodes
-    
+
     # Compile a XStringNode node
     alias visit_x_string_node visit_child_nodes
-    
+
     # Compile a YieldNode node
     alias visit_yield_node visit_child_nodes
   end

data/lib/prism/ffi.rb

@@ -234,11 +234,11 @@ module Prism
       loader = Serialize::Loader.new(source, buffer.read)
 
       tokens = loader.load_tokens
-      node, comments, errors, warnings = loader.load_nodes
+      node, comments, magic_comments, errors, warnings = loader.load_nodes
 
       tokens.each { |token,| token.value.force_encoding(loader.encoding) }
 
-      ParseResult.new([node, tokens], comments, errors, warnings, source)
+      ParseResult.new([node, tokens], comments, magic_comments, errors, warnings, source)
     end
   end
 

data/lib/prism/lex_compat.rb

@@ -370,10 +370,9 @@ module Prism
             @embexpr_balance -= 1
           when :on_tstring_content
             if embexpr_balance == 0
-              token.value.split(/(?<=\n)/).each_with_index do |line, index|
-                next if line.strip.empty? && line.end_with?("\n")
-                next if !(dedent_next || index > 0)
+              line = token.value
 
+              if !(line.strip.empty? && line.end_with?("\n")) && dedent_next
                 leading = line[/\A(\s*)\n?/, 1]
                 next_dedent = 0
 
@@ -430,6 +429,45 @@ module Prism
             return results
           end
 
+          # If the minimum common whitespace is 0, then we need to concatenate
+          # string nodes together that are immediately adjacent.
+          if dedent == 0
+            results = []
+            embexpr_balance = 0
+
+            index = 0
+            max_index = tokens.length
+
+            while index < max_index
+              token = tokens[index]
+              index += 1
+
+              case token.event
+              when :on_embexpr_beg, :on_heredoc_beg
+                embexpr_balance += 1
+                results << token
+              when :on_embexpr_end, :on_heredoc_end
+                embexpr_balance -= 1
+                results << token
+              when :on_tstring_content
+                if embexpr_balance == 0
+                  results << token
+
+                  while index < max_index && tokens[index].event == :on_tstring_content
+                    token.value << tokens[index].value
+                    index += 1
+                  end
+                else
+                  results << token
+                end
+              else
+                results << token
+              end
+            end
+
+            return results
+          end
+
           # Otherwise, we're going to run through each token in the list and
           # insert on_ignored_sp tokens for the amount of dedent that we need to
           # perform. We also need to remove the dedent from the beginning of
@@ -787,11 +825,7 @@ module Prism
       # We sort by location to compare against Ripper's output
       tokens.sort_by!(&:location)
 
-      if result_value.size - 1 > tokens.size
-        raise StandardError, "Lost tokens when performing lex_compat"
-      end
-
-      ParseResult.new(tokens, result.comments, result.errors, result.warnings, [])
+      ParseResult.new(tokens, result.comments, result.magic_comments, result.errors, result.warnings, [])
     end
   end