gloo 7.1.3 → 7.2

data/lib/gloo/persist/file_loader.rb

@@ -7,6 +7,11 @@
 # raw formatting) -- so a later save can rewrite the file instead of
 # regenerating it from scratch.
 #
+# Every byte of the file goes into exactly one source node, in file
+# order, so the document's text is the file. When the load finishes,
+# each declaration remembers its object's place and value, so a save
+# can tell what changed.
+#
 # Work is delegated to TriviaBuffer (comment/blank-line buffering),
 # ScriptBodyCollector (script bodies), IndentStack (nesting shared by
 # the heap and source trees), ShorthandExpander (nested-container
@@ -29,6 +34,10 @@ module Gloo
         'the line above; nesting it under that line.'.freeze
       MISALIGNED_WARNING = "Indentation doesn't line up with any line " \
         'above; nesting it under the closest one.'.freeze
+      INVALID_UTF8_WARNING = 'Line is not valid UTF-8; using U+FFFD for ' \
+        'the bytes that are not.'.freeze
+      NO_END_WARNING = "BEGIN block has no END; it runs to the end of the file.".freeze
+      CONTINUATION = "\\\n".freeze
 
       # A 'load lib {name}' (or 'load ext {name}') statement at the top
       # of a file, before the first object declaration. It makes a core
@@ -55,9 +64,9 @@ module Gloo
         @shorthand = Gloo::Persist::ShorthandExpander.new( engine )
         @ledger = Gloo::Persist::DeclarationLedger.new( engine )
         @in_block = false
-        @block_value = ''
+        @block_value = +''
+        @save_raw = nil
         @body_started = false
-        @debug = false
       end
 
       #
@@ -87,14 +96,14 @@ module Gloo
 
         @engine.log.debug "Loading file '#{@pn}'"
         @indent_stack = IndentStack.new( @engine.heap.root, @source_doc )
-        f = @mech.read( @pn )
+        f = @data = @mech.read( @pn )
 
         exec_env = @engine.exec_env
         outer_location = exec_env.load_location
         begin
-          each_joined_line( f ) do |line, number|
+          each_joined_line( f ) do |line, raw, number|
             exec_env.load_location = "#{@pn}:#{number}"
-            dispatch_line( line )
+            dispatch_line( valid_utf8( line ), raw )
           end
           finish
         ensure
@@ -114,27 +123,44 @@ module Gloo
       end
 
       #
-      # Yield each line of the file with its line number (counting from
-      # 1). A line ending in a backslash continues on the next line:
-      # the two are joined, dropping the backslash and the line break,
-      # and the joined line has the number of the line it starts on.
+      # Yield each line of the file, its exact source text, and its line
+      # number (counting from 1). A line ending in a backslash continues
+      # on the next line: the two are joined, dropping the backslash and
+      # the line break, and the joined line has the number of the line
+      # it starts on. Not inside a BEGIN/END block, whose lines are the
+      # value, exactly as written.
       #
       def each_joined_line( data )
         pending = nil
+        raw = nil
         start = 0
         data.each_line.with_index( 1 ) do |line, number|
           if pending
             pending << line
+            raw << line
           else
             pending = line.dup
+            raw = line.dup
             start = number
           end
-          next if pending.delete_suffix!( "\\\n" )
+          next if !@in_block && pending.delete_suffix!( CONTINUATION )
 
-          yield pending, start
+          yield pending, raw, start
           pending = nil
         end
-        yield pending, start if pending
+        yield pending, raw, start if pending
+      end
+
+      #
+      # A line to work with: as read if it's valid UTF-8, otherwise with
+      # U+FFFD in place of the bytes that aren't, and a warning. The
+      # source text keeps the original bytes.
+      #
+      def valid_utf8( line )
+        return line if line.valid_encoding?
+
+        @engine.warn INVALID_UTF8_WARNING
+        return line.scrub( "\uFFFD" )
       end
 
       # ---------------------------------------------------------------------
@@ -145,30 +171,49 @@ module Gloo
       # Route one line of the file to whichever mode we're currently in:
       # collecting a script body, collecting a BEGIN/END block, trivia
       # (comment/blank), a top-of-file directive, or a normal
-      # declaration.
+      # declaration. line is the line as read (continuations joined),
+      # raw its exact source text.
       #
-      def dispatch_line( line )
-        return handle_body_line( line ) if @body.active
-        return handle_block_line( line ) if @in_block
-        return handle_trivia_line( line ) if skip_line?( line )
+      def dispatch_line( line, raw )
+        return handle_body_line( line, raw ) if @body.active
+        return handle_block_line( line, raw ) if @in_block
+        return handle_trivia_line( line, raw ) if skip_line?( line )
 
         if !@body_started && line =~ LIB_DIRECTIVE
-          @indent_stack.node.children << Source::DirectiveNode.new( line.strip )
+          # Comments before the directive stay before it.
+          @trivia.flush_into( @indent_stack.node.children )
+          @indent_stack.node.children << Source::DirectiveNode.new( raw )
           run_lib_directive line
           return
         end
         @body_started = true
 
-        handle_declaration_line( line )
+        handle_declaration_line( line, raw )
       end
 
       #
       # End of file: close out anything still open so its content
-      # isn't silently dropped.
+      # isn't silently dropped, then have every declaration remember
+      # its object's place and value as loaded.
       #
       def finish
+        finish_open_block if @in_block
         @body.finish
-        @trivia.flush_into( @indent_stack.node.children )
+        # Comments and blank lines at the end of the file belong to the
+        # file, not to whatever declaration happened to come last.
+        @trivia.flush_into( @source_doc.children )
+        @source_doc.missing_final_newline! unless @data.empty? || @data.end_with?( "\n" )
+        @source_doc.obj_nodes.each( &:remember )
+      end
+
+      #
+      # A BEGIN block with no END: the best guess is that it runs to the
+      # end of the file, so the object still loads with that value.
+      #
+      def finish_open_block
+        @engine.warn NO_END_WARNING
+        @in_block = false
+        finalize_declaration( @save_line, @save_raw + @block_raw )
       end
 
       # ---------------------------------------------------------------------
@@ -187,11 +232,11 @@ module Gloo
       # resolving once the right moment comes (the next declaration, or
       # end of file).
       #
-      def handle_trivia_line( line )
+      def handle_trivia_line( line, raw )
         if line.strip.empty?
-          @trivia.push_blank( chomped( line ) )
+          @trivia.push_blank( raw )
         else
-          @trivia.push_comment( chomped( line ), tab_count( line ) )
+          @trivia.push_comment( raw, tab_count( line ) )
         end
       end
 
@@ -203,11 +248,11 @@ module Gloo
       # Inside a BEGIN/END block every line is literal content -- kept
       # exactly as-is, comments and blank lines included.
       #
-      def handle_block_line( line )
+      def handle_block_line( line, raw )
+        @block_raw << raw
         if line.strip == END_BLOCK
           @in_block = false
-          @block_end_indent = leading_ws( line )
-          finalize_declaration( @save_line )
+          finalize_declaration( @save_line, @save_raw + @block_raw )
         else
           @block_value << line
         end
@@ -221,9 +266,9 @@ module Gloo
       # One line while a script body might be open. Re-dispatches the
       # line normally if it turns out to have closed the body.
       #
-      def handle_body_line( line )
-        consumed = @body.handle_line( line, tab_count( line ) )
-        dispatch_line( line ) unless consumed
+      def handle_body_line( line, raw )
+        consumed = @body.handle_line( line, raw, tab_count( line ) )
+        dispatch_line( line, raw ) unless consumed
       end
 
       # ---------------------------------------------------------------------
@@ -233,25 +278,28 @@ module Gloo
       #
       # A line that starts (or is) an object declaration.
       #
-      def handle_declaration_line( line )
+      def handle_declaration_line( line, raw )
         if line.strip.end_with?( BEGIN_BLOCK )
           @in_block = true
           @save_line = line
+          @save_raw = raw
+          @block_raw = +''
           return
         end
 
-        finalize_declaration( line )
+        finalize_declaration( line, raw )
       end
 
       #
       # Place the declaration at its indentation level and create the
-      # object (and its Source::ObjNode).
+      # object (and its Source::ObjNode). raw is the declaration's exact
+      # text (with its BEGIN/END block, if it has one).
       #
-      def finalize_declaration( line )
+      def finalize_declaration( line, raw )
         line_tabs = tab_count( line )
         placed = @indent_stack.place( line_tabs, @last, @last_node )
         warn_indent( placed ) if placed
-        create_declared_obj( line, line_tabs )
+        create_declared_obj( line, raw, line_tabs )
       end
 
       #
@@ -284,8 +332,9 @@ module Gloo
       # Create the heap object and its source node for one declaration
       # line, and start body collection if its type calls for one.
       #
-      def create_declared_obj( line, line_tabs )
+      def create_declared_obj( line, raw, line_tabs )
         name, type, value, style = split_declaration( line )
+        style[ :repair ] ||= name.nil?
         name ||= placeholder_name( @indent_stack.parent )
         leaf, parent, roots, created = @shorthand.expand( name, @indent_stack.parent )
         roots.each { |r| @ledger.root( r ) }
@@ -298,7 +347,9 @@ module Gloo
         @ledger.created( @last )
         @ledger.root( @last ) if parent == @engine.heap.root
 
-        node = build_obj_node( leading_ws( line ), name, type, value, style )
+        node = build_obj_node( line, raw, name, type, value, style )
+        # A declaration gloo had to guess at is written as understood.
+        node.repaired_head = "#{node.raw_indent}#{name} [#{type}]" if style[ :repair ]
         node.leading_doc = @trivia.take_leading_doc( line_tabs, @indent_stack.node.children )
         # First non-empty doc wins, same as "first value wins" for a
         # name re-declared across files -- @last is the same object
@@ -329,29 +380,43 @@ module Gloo
         splitter = LineSplitter.new( line, @indent_stack&.tabs || 0 )
         name, type, value = splitter.split
         @engine.syntax_err "#{MISSING_BRACKET_ERR}'#{type}'." if splitter.missing_bracket
-        style = { :raw_tail => splitter.raw_tail, :block_style => :inline }
-        return name, type, value, style if @block_value == ''
+        style = { :raw_tail => splitter.raw_tail, :block_style => :inline,
+                  :repair => splitter.missing_bracket && closable?( splitter.raw_tail ) }
+        return name, type, value, style unless @save_raw
 
         value = @block_value
-        @block_value = ''
+        @block_value = +''
+        @save_raw = nil
         style[ :block_style ] = :begin_end
-        style[ :raw_end_indent ] = @block_end_indent
         return name, type, value, style
       end
 
       #
-      # Build the source node for one object declaration. raw_value is
-      # kept byte-exact with whatever was handed to the heap object's
-      # set_value (trailing newline included, for a BEGIN/END block) so
-      # a save can compare the two directly.
+      # Can a missing ']' be closed on save? Only when the value comes
+      # next ('b [int : 3'), not when a ']' does ('a [string ] :', where
+      # closing it would add a second).
       #
-      def build_obj_node( raw_indent, name, type, value, style )
+      def closable?( tail )
+        rest = tail.lstrip
+        return rest.empty? || rest.start_with?( ':' )
+      end
+
+      #
+      # Build the source node for one object declaration: its exact text
+      # (raw), and its head -- the declaration line up to its value
+      # (indent, name and [type], as written) -- for a save that only
+      # changes the value. raw_value is what was handed to the heap
+      # object's set_value.
+      #
+      def build_obj_node( line, raw, name, type, value, style )
         node = Source::ObjNode.new( :name => name, :raw_type => type )
-        node.raw_indent = raw_indent
+        first = line.lines.first.to_s.chomp
+        node.raw_indent = leading_ws( first )
         node.raw_tail = style[ :raw_tail ]
+        node.head = first[ 0, first.length - node.raw_tail.length ]
         node.block_style = style[ :block_style ]
         node.raw_value = value
-        node.raw_end_indent = style[ :raw_end_indent ] || ''
+        node.raw = raw.dup
         node.obj = @last
         return node
       end
@@ -369,20 +434,21 @@ module Gloo
       end
 
       #
-      # Get the number of leading tabs.
+      # Get the indentation level of a line: a tab or SPACE_CNT spaces is
+      # one level, and they can be mixed, on one line or across lines.
       #
       def tab_count( line )
-        i = 0
-
-        if line[ i ] == ' '
-          i += 1 while line[ i ] == ' '
-          tab_equiv = ( i / SPACE_CNT ).to_i
-          puts "Found #{i} spaces => #{tab_equiv}" if @debug
-          return tab_equiv
+        width = 0
+        line.each_char do |c|
+          if c == ' '
+            width += 1
+          elsif c == "\t"
+            width += SPACE_CNT
+          else
+            break
+          end
         end
-
-        i += 1 while line[ i ] == "\t"
-        return i
+        return width / SPACE_CNT
       end
 
       #
@@ -400,13 +466,6 @@ module Gloo
         return line[ /\A[ \t]*/ ]
       end
 
-      #
-      # A line's raw text with its trailing newline removed.
-      #
-      def chomped( line )
-        return line.chomp( "\n" )
-      end
-
     end
   end
 end