gloo 6.3.0 → 6.4.0

data/lib/gloo/persist/file_loader.rb

@@ -1,7 +1,18 @@
 # Author::    Eric Crane  (mailto:eric.crane@mac.com)
 # Copyright:: Copyright (c) 2019 Eric Crane.  All rights reserved.
 #
-# Helper class used to load a file and create objects in the heap.
+# Helper class used to load a file, create objects in the heap, and
+# build a parallel Source::SourceDoc capturing everything about the
+# file that the heap alone can't represent (comments, blank lines,
+# raw formatting) -- so a later save can rewrite the file instead of
+# regenerating it from scratch.
+#
+# Work is delegated to CommentBuffer (comment buffering),
+# ScriptBodyCollector (script bodies), IndentStack (nesting shared by
+# the heap and source trees), ShorthandExpander (nested-container
+# shorthand), and DeclarationLedger (this file's roots + cross-file
+# clash warnings) -- this class is the orchestrator: per-line dispatch
+# and BEGIN/END handling.
 #
 
 module Gloo
@@ -20,7 +31,7 @@ module Gloo
       # afterward.
       LIB_DIRECTIVE = /\A(?:load|ld)\s+(?:lib|ext)\s+\S+\s*\z/i.freeze
 
-      attr_reader :obj
+      attr_reader :obj, :source_doc
 
       #
       # Set up a file storage for an object.
@@ -29,16 +40,25 @@ module Gloo
         @engine = engine
         @mech = @engine.platform.get_file_mech( @engine )
         @pn = pn
-        @tabs = 0
         @obj = nil
-        @in_multiline = false
-        @exiting_multiline = false
+        @source_doc = Gloo::Persist::Source::SourceDoc.new
+        @comments = Gloo::Persist::CommentBuffer.new
+        @body = Gloo::Persist::ScriptBodyCollector.new
+        @shorthand = Gloo::Persist::ShorthandExpander.new( engine )
+        @ledger = Gloo::Persist::DeclarationLedger.new( engine, pn )
         @in_block = false
         @block_value = ''
         @body_started = false
         @debug = false
       end
 
+      #
+      # The top-level objects this file declared (for save resolution).
+      #
+      def roots
+        return @ledger.roots
+      end
+
       #
       # Load the objects from the file.
       #
@@ -49,25 +69,12 @@ module Gloo
         end
 
         @engine.log.debug "Loading file '#{@pn}'"
-        @tabs = 0
-        @parent_stack = []
-        @parent = @engine.heap.root
-        @parent_stack.push @parent
+        @indent_stack = IndentStack.new( @engine.heap.root, @source_doc )
         f = @mech.read( @pn )
 
         f = join_continuations( f )
-
-        f.each_line do |line|
-          next if skip_line? line
-
-          if !@body_started && line =~ LIB_DIRECTIVE
-            run_lib_directive line
-            next
-          end
-          @body_started = true
-
-          handle_one_line line
-        end
+        f.each_line { |line| dispatch_line( line ) }
+        finish
       end
 
       #
@@ -88,115 +95,195 @@ module Gloo
         return data.gsub( "\\\n", '' )
       end
 
+      # ---------------------------------------------------------------------
+      #    Per-line dispatch
+      # ---------------------------------------------------------------------
+
       #
-      # Process one one of the file we're loading.
+      # Route one line of the file to whichever mode we're currently in:
+      # collecting a script body, collecting a BEGIN/END block, trivia
+      # (comment/blank), a top-of-file directive, or a normal
+      # declaration.
       #
-      def handle_one_line( line )
-        if line.strip.end_with? BEGIN_BLOCK
-          @in_block = true
-          @save_line = line
-        elsif @in_block
-          if line.strip == END_BLOCK
-            @in_block = false
-            determine_indent @save_line
-            process_line @save_line
-          else
-            @block_value << line
-          end
-        else
-          determine_indent line
-          process_line line
+      def dispatch_line( line )
+        return handle_body_line( line ) if @body.active
+        return handle_block_line( line ) if @in_block
+        return handle_trivia_line( line ) if skip_line?( line )
+
+        if !@body_started && line =~ LIB_DIRECTIVE
+          @indent_stack.node.children << Source::DirectiveNode.new( line.strip )
+          run_lib_directive line
+          return
         end
+        @body_started = true
+
+        handle_declaration_line( line )
       end
 
       #
-      # Is this line a comment or a blank line?
-      # If so we'll skip it.
+      # End of file: close out anything still open so its content
+      # isn't silently dropped.
       #
-      def skip_line?( line )
-        line = line.strip
-        return true if line.empty?
-        return true if line[ 0 ] == '#'
-
-        return false
+      def finish
+        @body.finish
+        @comments.flush_into( @indent_stack.node.children )
       end
 
+      # ---------------------------------------------------------------------
+      #    Trivia (comments and blank lines)
+      # ---------------------------------------------------------------------
+
       #
-      # Determine the relative indent level for the line.
+      # A comment or blank line, outside of any block/body. A comment
+      # is buffered -- it may turn out to be the leading_doc for the
+      # declaration that follows. A blank line always breaks that
+      # association (detaches any buffered comments as floating nodes)
+      # and is itself kept, not discarded.
       #
-      def determine_indent( line )
-        tabs = tab_count( line )
-        @indent = 0 # same level as prior line
-        if tabs > @tabs # indent
-          # TODO:  What if indent is more than one more level?
-          @tabs = tabs
-          @indent = 1
-        elsif tabs < @tabs # outdent
-          diff = @tabs - tabs
-          @tabs -= diff
-          @indent -= diff
+      def handle_trivia_line( line )
+        if line.strip.empty?
+          @comments.flush_into( @indent_stack.node.children )
+          @indent_stack.node.children << Source::BlankNode.new( chomped( line ) )
+        else
+          @comments.push( chomped( line ), tab_count( line ) )
         end
-        puts "tabs: #{@tabs}, indent: #{@indent}, line: #{line}" if @debug
       end
 
+      # ---------------------------------------------------------------------
+      #    BEGIN / END blocks
+      # ---------------------------------------------------------------------
+
       #
-      # Process one line and add objects.
+      # Inside a BEGIN/END block every line is literal content -- kept
+      # exactly as-is, comments and blank lines included.
       #
-      def process_line( line )
-        # reset multiline unless we're actually indented
-        if @in_multiline && @multi_indent > @indent
-          puts "Done multiline mi: #{@multi_indent}, i: #{@indent}" if @debug
-          @in_multiline = false
-          @exiting_multiline = true
-        end
-
-        if @in_multiline
-          @last.add_line line
+      def handle_block_line( line )
+        if line.strip == END_BLOCK
+          @in_block = false
+          @block_end_indent = leading_ws( line )
+          finalize_declaration( @save_line )
         else
-          setup_process_obj_line
-          process_obj_line line
+          @block_value << line
         end
       end
 
+      # ---------------------------------------------------------------------
+      #    Script bodies (indented lines, no BEGIN/END)
+      # ---------------------------------------------------------------------
+
       #
-      # Setup and get ready to process an object line.
+      # One line while a script body might be open. Re-dispatches the
+      # line normally if it turns out to have closed the body.
       #
-      def setup_process_obj_line
-        if @exiting_multiline
-          @exiting_multiline = false
-          @indent += 1
-        end
+      def handle_body_line( line )
+        consumed = @body.handle_line( line, tab_count( line ) )
+        dispatch_line( line ) unless consumed
+      end
 
-        if @indent.positive?
-          @parent = @last
-          @parent_stack.push @parent
-        elsif @indent.negative?
-          @indent.abs.times do
-            @parent_stack.pop
-            @parent = @parent_stack.last
-          end
+      # ---------------------------------------------------------------------
+      #    Object declarations
+      # ---------------------------------------------------------------------
+
+      #
+      # A line that starts (or is) an object declaration.
+      #
+      def handle_declaration_line( line )
+        if line.strip.end_with?( BEGIN_BLOCK )
+          @in_block = true
+          @save_line = line
+          return
         end
+
+        finalize_declaration( line )
       end
 
       #
-      # Process one line and add objects.
+      # Place the declaration at its indentation level and create the
+      # object (and its Source::ObjNode).
       #
-      def process_obj_line( line )
-        name, type, value = split_line( line )
-        unless @block_value == ''
-          value = @block_value
-          @block_value = ''
-        end
-        params = { name: name, type: type, value: value, parent: @parent }
+      def finalize_declaration( line )
+        line_tabs = tab_count( line )
+        @indent_stack.place( line_tabs, @last, @last_node )
+        create_declared_obj( line, line_tabs )
+      end
+
+      #
+      # Create the heap object and its source node for one declaration
+      # line, and start body collection if its type calls for one.
+      #
+      def create_declared_obj( line, line_tabs )
+        name, type, value, style = split_declaration( line )
+        leaf, parent, roots, created = @shorthand.expand( name, @indent_stack.parent )
+        roots.each { |r| @ledger.root( r ) }
+        created.each { |c| @ledger.created( c ) }
+
+        prior = parent.find_child( leaf )
+        params = { :name => leaf, :type => type, :value => value, :parent => parent }
         @last = @engine.factory.create( params )
+        @ledger.clash( prior, leaf, value ) if prior
+        @ledger.created( @last )
+        @ledger.root( @last ) if parent == @engine.heap.root
 
-        if value&.empty? && @last&.multiline_value?
-          @multi_indent = 0
-          @in_multiline = true
-          puts "*** Start multiline. multi_indent: #{@multi_indent}" if @debug
-        end
+        node = build_obj_node( leading_ws( line ), name, type, value, style )
+        node.leading_doc = @comments.take_leading_doc( line_tabs, @indent_stack.node.children )
+        @indent_stack.node.children << node
+        @last_node = node
+        @obj ||= @ledger.roots.last || @last
+
+        # A whitespace-only inline value (trailing spaces after the ':'
+        # on a `script :` line) counts as no value -- start the body.
+        no_value = value.nil? || value.strip.empty?
+        @body.start( node, @last, @indent_stack.tabs ) if no_value && @last&.multiline_value?
+      end
+
+      #
+      # Split a declaration line into name/type/value, plus a style
+      # hash (:block_style, :raw_tail), folding in any BEGIN/END block
+      # value collected just before it. raw_tail is the exact source
+      # text that followed the type bracket on this line -- eg.
+      # " : BEGIN" or " :" -- kept for a save to reproduce the
+      # declaration's original spacing when unchanged.
+      #
+      def split_declaration( line )
+        splitter = LineSplitter.new( line, @indent_stack&.tabs || 0 )
+        name, type, value = splitter.split
+        style = { :raw_tail => splitter.raw_tail, :block_style => :inline }
+        return name, type, value, style if @block_value == ''
+
+        value = @block_value
+        @block_value = ''
+        style[ :block_style ] = :begin_end
+        style[ :raw_end_indent ] = @block_end_indent
+        return name, type, value, style
+      end
+
+      #
+      # Build the source node for one object declaration. raw_value is
+      # kept byte-exact with whatever was handed to the heap object's
+      # set_value (trailing newline included, for a BEGIN/END block) so
+      # a save can compare the two directly.
+      #
+      def build_obj_node( raw_indent, name, type, value, style )
+        node = Source::ObjNode.new( :name => name, :raw_type => type )
+        node.raw_indent = raw_indent
+        node.raw_tail = style[ :raw_tail ]
+        node.block_style = style[ :block_style ]
+        node.raw_value = value
+        node.raw_end_indent = style[ :raw_end_indent ] || ''
+        node.obj = @last
+        return node
+      end
+
+      #
+      # Is this line a comment or a blank line?
+      # If so we'll skip it.
+      #
+      def skip_line?( line )
+        line = line.strip
+        return true if line.empty?
+        return true if line[ 0 ] == '#'
 
-        @obj = @last if @obj.nil?
+        return false
       end
 
       #
@@ -220,10 +307,24 @@ module Gloo
       # Split the line into 3 parts.
       #
       def split_line( line )
-        o = LineSplitter.new( line, @tabs )
+        o = LineSplitter.new( line, @indent_stack&.tabs || 0 )
         return o.split
       end
 
+      #
+      # The line's leading whitespace, exactly as written.
+      #
+      def leading_ws( line )
+        return line[ /\A[ \t]*/ ]
+      end
+
+      #
+      # A line's raw text with its trailing newline removed.
+      #
+      def chomped( line )
+        return line.chomp( "\n" )
+      end
+
     end
   end
 end

data/lib/gloo/persist/file_saver.rb

@@ -3,26 +3,52 @@
 #
 # Helper class used to save an object to a file.
 #
+# Given a Source::SourceDoc (captured at load time), this is a
+# rewriter, not a regenerator: trivia (comments, blank lines,
+# directives) is emitted verbatim, and a declaration's raw text is
+# kept as-is unless the live object's value has actually changed --
+# only then is the value re-rendered, via the object's own
+# serialize_value. Deleted objects drop out; objects created since
+# load are rendered fresh and appended.
+#
+# With no SourceDoc at all (the object was never loaded from a file),
+# this falls back to plain regeneration from the heap, so saving a
+# brand-new in-memory object still works.
+#
 
 module Gloo
   module Persist
     class FileSaver
 
       #
-      # Set up a file storage for an object.
+      # Set up a file saver. source_doc is optional -- when given, the
+      # save rewrites it in place; otherwise the file is regenerated
+      # from the heap.
+      #
+      # For a multi-file namespace to round-trip, batch carries two sets
+      # (PersistMan#save_batch builds them):
+      #  - :others -- objects declared by *other* loaded files'
+      #    SourceDocs; this saver leaves them out even though they're
+      #    live children of a container it shares.
+      #  - :claimed -- a set shared across the whole save batch; a
+      #    brand-new object no file owns is written by the first saver to
+      #    reach it, which adds it here so the rest skip it.
       #
-      def initialize( engine, pn, obj )
+      def initialize( engine, pn, obj, source_doc = nil, batch = nil )
         @engine = engine
         @mech = @engine.platform.get_file_mech( @engine )
         @pn = pn
         @obj = obj
+        @source_doc = source_doc
+        @others = batch && batch[ :others ]
+        @claimed = batch && batch[ :claimed ]
       end
 
       #
       # Save the object to the file.
       #
       def save
-        data = get_obj( @obj )
+        data = @source_doc ? rewrite : regenerate
         @mech.write( @pn, data )
       end
 
@@ -34,18 +60,237 @@ module Gloo
       end
 
       #
-      # Convert an object to textual representation.
-      # This is a recursive function.
+      # Would saving now actually change anything on disk? True if any
+      # known declaration's value no longer matches its raw source, or
+      # its object was deleted. (Doesn't detect a brand-new object with
+      # no source node of its own -- an approximation, not a guarantee,
+      # good enough to warn before a reload discards changes.)
+      #
+      def dirty?
+        return false unless @source_doc
+
+        return any_dirty?( @source_doc.children, @engine.heap.root )
+      end
+
+      #
+      # Convert an object to textual representation from scratch, with
+      # no source text to preserve. This is a recursive function, and
+      # is also reused by the rewriter for any object created since
+      # load (which has no source node of its own to rewrite).
       #
       def get_obj( obj, indent = 0 )
         t = tabs( indent )
-        str = "#{t}#{obj.name} [#{obj.type_display}] : #{obj.value_display}\n"
+        str = "#{t}#{obj.name} [#{obj.type_display}]#{obj.serialize_value( indent )}\n"
         obj.children.each do |child|
           str << get_obj( child, indent + 1 )
         end
         return str
       end
 
+      private
+
+      #
+      # Regenerate the whole file from the heap (the pre-Source::SourceDoc
+      # behavior), used when there's no source text to rewrite.
+      #
+      def regenerate
+        return get_obj( @obj )
+      end
+
+      #
+      # Does any node in the given list represent a change from what's
+      # on disk -- its object gone from live_parent's children (an
+      # untracked deletion counts too), or its value changed?
+      #
+      def any_dirty?( nodes, live_parent )
+        nodes.any? do |node|
+          next false unless node.is_a?( Source::ObjNode )
+
+          _top, leaf = locate( node, live_parent )
+          next true unless leaf
+          next true unless value_unchanged?( node )
+
+          any_dirty?( node.children, leaf )
+        end
+      end
+
+      # ---------------------------------------------------------------------
+      #    Rewrite (Source::SourceDoc-driven)
+      # ---------------------------------------------------------------------
+
+      #
+      # Rewrite the file from its source document: known declarations
+      # are patched in place, deleted ones dropped, and anything new
+      # appended.
+      #
+      def rewrite
+        return render_children( @source_doc.children, @engine.heap.root, 0 )
+      end
+
+      #
+      # Render one level of source nodes against the live children of
+      # live_parent: trivia passes through untouched; a declaration
+      # whose object is still present is patched in place; one whose
+      # object is gone is dropped; anything in live_parent.children
+      # with no matching source node is new and gets rendered fresh
+      # (unless another file owns it -- see #owned).
+      #
+      def render_children( nodes, live_parent, indent )
+        str = ''
+        seen = []
+        nodes.each do |node|
+          if node.is_a?( Source::ObjNode )
+            top, leaf = locate( node, live_parent )
+            next unless leaf
+
+            seen << top
+            str << render_obj_node( node, indent )
+          else
+            str << render_trivia( node )
+          end
+        end
+
+        ( live_parent.children - seen ).each do |child|
+          next unless write_new_child?( child )
+
+          str << get_obj( child, indent )
+        end
+        return str
+      end
+
+      #
+      # A live child with no source node at this level. Write it unless
+      # another file owns it, or another file in this batch already
+      # wrote it as a brand-new object. A genuinely new object is
+      # written here and claimed.
+      #
+      def write_new_child?( child )
+        return false if @others&.key?( child )
+        return false if @claimed&.key?( child )
+
+        @claimed[ child ] = true if @claimed
+        return true
+      end
+
+      #
+      # Resolve a source node against the live tree under live_parent.
+      # A plain name is a direct child; a dotted name (nested container
+      # shorthand, page.core.users.list) walks the chain. Returns
+      # [top-level object to mark as handled, the node's own object],
+      # or [nil, nil] if the node's object is no longer there.
+      #
+      def locate( node, live_parent )
+        return [ nil, nil ] unless node.obj
+
+        segments = node.name.split( '.' )
+        if segments.length == 1
+          return live_parent.children.include?( node.obj ) ? [ node.obj, node.obj ] : [ nil, nil ]
+        end
+
+        top = live_parent.find_child( segments.first )
+        return [ nil, nil ] unless top
+
+        leaf = top
+        segments[ 1..-1 ].each do |seg|
+          leaf = leaf.find_child( seg )
+          return [ nil, nil ] unless leaf
+        end
+        return leaf.equal?( node.obj ) ? [ top, leaf ] : [ nil, nil ]
+      end
+
+      #
+      # A comment, blank line, or top-of-file directive -- passed
+      # through exactly as read.
+      #
+      def render_trivia( node )
+        return "#{node.raw}\n"
+      end
+
+      #
+      # One declaration whose object is still live: its leading_doc (if
+      # any) and its own line are kept (name/type/indent never change
+      # once declared), its value is kept raw or re-rendered depending
+      # on whether it changed, and its children are rewritten the same
+      # way, one level deeper.
+      #
+      def render_obj_node( node, indent )
+        str = node.leading_doc ? "#{node.leading_doc}\n" : ''
+        str << "#{node.raw_indent}#{node.name} [#{node.raw_type}]#{declaration_tail( node, indent )}\n"
+        str << render_children( node.children, node.obj, indent + 1 )
+        return str
+      end
+
+      #
+      # The text that follows "name [type]" on the declaration line:
+      # the raw source, unless the live value no longer matches it.
+      #
+      def declaration_tail( node, indent )
+        return raw_declaration_tail( node ) if value_unchanged?( node )
+
+        return node.obj.serialize_value( indent )
+      end
+
+      #
+      # Reproduce the original declaration's value text exactly as it
+      # was read. raw_tail is the declaration line's own exact tail
+      # (eg. " : BEGIN" or " :"); a begin_end/body style has further
+      # raw content following it.
+      #
+      def raw_declaration_tail( node )
+        case node.block_style
+        when :begin_end
+          return "#{node.raw_tail}\n#{node.raw_value}#{node.raw_end_indent}END"
+        when :body
+          return node.raw_value.to_s.empty? ? node.raw_tail : "#{node.raw_tail}\n#{node.raw_value}"
+        else
+          return node.raw_tail
+        end
+      end
+
+      # ---------------------------------------------------------------------
+      #    Has the value actually changed since load?
+      # ---------------------------------------------------------------------
+
+      #
+      # Does the live object's value still match what raw_value would
+      # produce? A script body is compared line by line (raw comments
+      # excluded, since they were never run); everything else is
+      # compared by re-running the raw text through the same set_value
+      # conversion the loader used, and comparing the result.
+      #
+      def value_unchanged?( node )
+        return false unless node.obj
+        return body_value_unchanged?( node ) if node.block_style == :body
+
+        trial = node.obj.class.new( @engine )
+        trial.set_value( node.raw_value )
+        return trial.value == node.obj.value
+      rescue
+        return false
+      end
+
+      #
+      # A script body's raw text, minus its comment lines (never
+      # executed) and with each line stripped, should equal the live
+      # object's own command lines.
+      #
+      def body_value_unchanged?( node )
+        raw_lines = node.raw_value.to_s.split( "\n" )
+        executable = raw_lines.reject { |l| l.strip.start_with?( '#' ) }.map( &:strip )
+        return executable == current_script_lines( node.obj )
+      end
+
+      #
+      # The live object's command lines, whatever form its value is
+      # currently in.
+      #
+      def current_script_lines( obj )
+        return obj.value if obj.value_is_array?
+        return [] if obj.value_is_blank?
+
+        return [ obj.value.to_s.strip ]
+      end
+
     end
   end
 end