fig 0.1.81 → 0.2.1

data/lib/fig/string_tokenizer.rb

@@ -1,11 +1,33 @@
+# coding: utf-8
+
 require 'fig/tokenized_string'
 require 'fig/tokenized_string/plain_segment'
 
 module Fig; end
 
 class Fig::StringTokenizer
-  def initialize(subexpression_matchers = DEFAULT_SUBEXPRESSION_MATCHER)
+  # subexpression_matchers is an array of hashes.  Each hash is expected to
+  # contain two keys: :pattern and :action.
+  #
+  # The :pattern value needs to be a regular expression for the substring that
+  # needs special handling.
+  #
+  # The :action value needs to be a block that takes two parameters.
+  #
+  # The first parameter is the text that was matched and the second is the
+  # error block passed to #tokenize().
+  #
+  # On success the block returns either a String containing replacement text or
+  # a Fig::TokenizedString::Token representing the special handling of the
+  # consumed text.  If there was a problem, then the error block should have
+  # been invoked and the block should return nil.
+  #
+  #
+  # metacharacters is a regular expression character class for characters that
+  # need to be escaped when un-single quoting a string.
+  def initialize(subexpression_matchers = [], metacharacters = '')
     @subexpression_matchers = subexpression_matchers
+    @metacharacters         = metacharacters
 
     return
   end
@@ -28,15 +50,11 @@ class Fig::StringTokenizer
 
     return if @segments.empty?
 
-    return Fig::TokenizedString.new(@segments, @single_quoted)
+    return Fig::TokenizedString.new(@segments, @single_quoted, @metacharacters)
   end
 
   private
 
-  DEFAULT_SUBEXPRESSION_MATCHER = [
-    { :pattern => %r<\@>, :action => lambda {|character| character} }
-  ]
-
   def strip_quotes_and_process_escapes()
     if @string.length == 0
       @single_quoted = false
@@ -60,13 +78,15 @@ class Fig::StringTokenizer
 
   def strip_single_quotes_and_process_escapes()
     return false if @string[0..0] != %q<'> && @string[-1..-1] != %q<'>
-    return false if @string =~ %r< \A (?: \\{2} )* \\ ' \z >x # «\'» is legal
+    return false if @string =~ %r< # «\'» is legal
+      \A ( [^\\']* (?: \\{2} )* \\ ' )* \z
+    >x
 
     if (
-      @string.length  == 1                         ||
-      @string[0..0]   != %q<'>                     ||
-      @string[-1..-1] != %q<'>                     ||
-      @string =~ %r< [^\\] (?: \\{2} )* \\ ' \z >x
+      @string.length  == 1                                      ||
+      @string[0..0]   != %q<'>                                  ||
+      @string[-1..-1] != %q<'>                                  ||
+      @string =~ %r< [^\\] (?: \\{2} )* (?: \\ | ' .* ) ' \z >x
     )
       @error_block.call 'has unbalanced single quotes.'
       return
@@ -85,7 +105,8 @@ class Fig::StringTokenizer
   end
 
   def strip_double_quotes_and_process_escapes()
-    return if ! check_and_strip_double_quotes
+    was_quoted = check_and_strip_double_quotes
+    return if was_quoted.nil?
 
     if @string == %q<\\'>
       @segments << Fig::TokenizedString::PlainSegment.new(%q<'>)
@@ -93,7 +114,7 @@ class Fig::StringTokenizer
       return
     end
 
-    generate_segments
+    generate_segments was_quoted
 
     return
   end
@@ -101,7 +122,7 @@ class Fig::StringTokenizer
   def check_and_strip_double_quotes()
     # We accept any unquoted single character at this point.  Later validation
     # will catch bad characters.
-    return true if @string =~ %r< \A \\ . \z >xm
+    return false if @string =~ %r< \A \\ . \z >xm
 
     if @string[0..0] == %q<">
       if @string.length == 1 || @string[-1..-1] != %q<">
@@ -115,16 +136,18 @@ class Fig::StringTokenizer
       end
 
       @string.sub!( %r< \A " (.*) " \z >xm, '\1' )
+
+      return true
     elsif @string =~ %r< (?: \A | [^\\] ) (?: \\{2} )* " \z >xm
       @error_block.call \
         %q<has unbalanced double quotes; it ends in a double quote when it didn't start with one.>
       return
     end
 
-    return true
+    return false
   end
 
-  def generate_segments()
+  def generate_segments(was_quoted)
     plain_string = nil
 
     while ! @string.empty?
@@ -135,13 +158,20 @@ class Fig::StringTokenizer
             @error_block.call 'ends in an incomplete escape.'
             return
           end
-          if subexpression_match(remainder) || remainder[0..0] == %q<">
+          subexpression_matched = subexpression_match(remainder)
+          return if subexpression_matched.nil?
+          if (
+            subexpression_matched                       ||
+            remainder[0..0] == %q<">                    ||
+            ! was_quoted && remainder[0..0] == %q<'>
+          )
             plain_string ||= ''
             plain_string << slashes
             plain_string << remainder[0..0]
             @string = remainder[1..-1] || ''
           else
-            @error_block.call "contains a bad escape sequence (\\#{$1})."
+            @error_block.call \
+              "contains a bad escape sequence (\\#{remainder[0..0]})."
             return
           end
         else
@@ -151,17 +181,23 @@ class Fig::StringTokenizer
         end
       else
         replacement, remainder = subexpression_match @string
+        return if replacement.nil?
         if replacement
           if replacement.is_a? String
             plain_string << replacement
           else
-            @segments << Fig::TokenizedString::PlainSegment.new(plain_string)
-            plain_string = nil
+            if ! plain_string.nil?
+              @segments << Fig::TokenizedString::PlainSegment.new(plain_string)
+              plain_string = nil
+            end
+            @segments << replacement
           end
           @string = remainder
-        elsif @string =~ %r< \A (["']) >xm # Fix single quotes in quoted strings
-          quote_name = $1 == %q<'> ? 'single' : 'double'
-          @error_block.call "contains an unescaped #{quote_name} quote."
+        elsif @string =~ %r< \A " >xm
+          @error_block.call 'contains an unescaped double quote.'
+          return
+        elsif ! was_quoted && @string =~ %r< \A ' >xm
+          @error_block.call 'contains an unescaped single quote.'
           return
         else
           plain_string ||= ''
@@ -185,11 +221,13 @@ class Fig::StringTokenizer
       pattern = matcher[:pattern]
       if sub_string =~ %r< \A ( #{pattern} ) >x
         subexpression, remainder = $1, $'
-        replacement = matcher[:action].call subexpression
+        replacement = matcher[:action].call subexpression, @error_block
+
+        return if ! replacement
         return [replacement, remainder]
       end
     end
 
-    return
+    return false
   end
 end

data/lib/fig/tokenized_string.rb

@@ -1,9 +1,10 @@
 module Fig; end
 
 class Fig::TokenizedString
-  def initialize(segments, single_quoted)
-    @segments      = segments
-    @single_quoted = single_quoted
+  def initialize(segments, single_quoted, metacharacters)
+    @segments       = segments
+    @single_quoted  = single_quoted
+    @metacharacters = metacharacters
 
     return
   end
@@ -12,11 +13,36 @@ class Fig::TokenizedString
     return @single_quoted
   end
 
-  def to_expanded_string()
-    return ( @segments.collect {|segment| segment.to_expanded_string} ).join ''
+  def can_be_single_quoted?()
+    return true if single_quoted?
+    return @segments.all? {|segment| segment.type.nil?}
+  end
+
+  def to_expanded_string(&block)
+    return (
+      @segments.collect { |segment| segment.to_expanded_string(&block) }
+    ).join ''
   end
 
   def to_escaped_string()
     return ( @segments.collect {|segment| segment.to_escaped_string} ).join ''
   end
+
+  def to_single_quoted_string()
+    return to_escaped_string if single_quoted?
+
+    return (
+      @segments.collect {|segment| segment.to_single_quoted_string}
+    ).join ''
+  end
+
+  def to_double_quotable_string()
+    return to_escaped_string if ! single_quoted?
+
+    return (
+      @segments.collect {
+        |segment| segment.to_double_quotable_string @metacharacters
+      }
+    ).join ''
+  end
 end

data/lib/fig/tokenized_string/plain_segment.rb

@@ -11,14 +11,44 @@ class Fig::TokenizedString::PlainSegment
   end
 
   def type
-    return :plain_segment
+    return nil
   end
 
-  def to_expanded_string()
+  def to_expanded_string(&block)
     return @raw_value.gsub(%r< \\ (.) >xm, '\1')
   end
 
   def to_escaped_string()
     return @raw_value
   end
+
+  # Should not be invoked if original string was single quoted.
+  def to_single_quoted_string()
+    # Raw value will have come from a non-single quoted string, so we unescape
+    # everything (including backslashes) and then escape backslashes and single
+    # quotes (which cannot be escaped outside of single quotes).
+    return \
+      @raw_value.gsub(%r< \\ (.) >xm, '\1').gsub(%r< ([\\']) >xm, '\\\\\1')
+  end
+
+  # Should not be invoked if original string was not single quoted.
+  def to_double_quotable_string(metacharacters)
+    quoted_value = @raw_value.gsub %r< ( ["#{metacharacters}] ) >xm, '\\\\\1'
+
+    quoted_value.gsub!(
+      %r<
+        (
+          (?: ^ | [^\\] ) # New line or non-backslash
+          (\\{2})*        # Even number of backslashes
+        )
+
+        # All single quotes must have been escaped.  The important bit is to
+        # not lose escaped backslashes.
+        \\'
+      >xm,
+      %q<\1'>
+    )
+
+    return quoted_value
+  end
 end

data/lib/fig/tokenized_string/token.rb

@@ -12,7 +12,19 @@ class Fig::TokenizedString::Token
     return
   end
 
+  def to_expanded_string(&block)
+    return block.call self
+  end
+
   def to_escaped_string()
     return raw_value
   end
+
+  def to_double_quotable_string(metacharacters)
+    return raw_value
+  end
+
+  def to_single_quoted_string()
+    raise NotImplementedError.new 'Cannot single-quote a token.'
+  end
 end

data/lib/fig/unparser.rb

@@ -1,3 +1,6 @@
+# Note: we very specifically do not require the files containing the
+# Unparser classes in order to avoid circular dependencies.
+
 module Fig; end
 
 module Fig::Unparser
@@ -7,8 +10,6 @@ module Fig::Unparser
   def self.class_for_statements(
     statements, emit_as_input_or_to_be_published_values
   )
-    # Note: we very specifically do not require the files containing the
-    # Unparser classes in order to avoid circular dependencies.
     statements = [statements].flatten
 
     versions =
@@ -21,12 +22,6 @@ module Fig::Unparser
       return Fig::Unparser::V0, explanations
     end
 
-    # TODO: Until v1 grammar handling is done, ensure we don't emit anything
-    # old fig versions cannot handle.
-    if ! ENV['FIG_ALLOW_NON_V0_GRAMMAR']
-      raise 'Reached a point where something could not be represented by the v0 grammar. Bailing out.'
-    end
-
     return Fig::Unparser::V1, explanations
   end
 
@@ -44,8 +39,11 @@ module Fig::Unparser
   private
 
   def self.gather_versions(statements, emit_as_input_or_to_be_published_values)
+    all_statements = gather_all_statements statements
+
     if emit_as_input_or_to_be_published_values == :emit_as_input
-      return statements.map {
+      versions = []
+      return all_statements.map {
         |statement|
 
         self.expand_version_and_explanation(
@@ -54,7 +52,7 @@ module Fig::Unparser
       }
     end
 
-    return statements.map {
+    return all_statements.map {
       |statement|
 
       self.expand_version_and_explanation(
@@ -63,6 +61,21 @@ module Fig::Unparser
     }
   end
 
+  def self.gather_all_statements(statements)
+    all_statements = []
+    statements.each do
+      |statement|
+
+      all_statements << statement
+
+      if statement.is_a? Fig::Statement::Configuration
+        all_statements << statement.statements
+      end
+    end
+
+    return all_statements.flatten
+  end
+
   def self.expand_version_and_explanation(statement, version_info)
     version, explanation = *version_info
     if explanation.nil?
@@ -159,6 +172,8 @@ module Fig::Unparser
   end
 
   def path(statement)
+    # I'd really like to change this to "add", but because the command-line
+    # option is "--append", I'm not going to.
     environment_variable(statement, 'append')
 
     return
@@ -202,8 +217,8 @@ module Fig::Unparser
     raise NotImplementedError
   end
 
-  def add_indent()
-    @text << @indent_string * @indent_level
+  def add_indent(indent_level = @indent_level)
+    @text << @indent_string * indent_level
 
     return
   end

data/lib/fig/unparser/v0.rb

@@ -1,4 +1,3 @@
-require 'fig/package_descriptor'
 require 'fig/unparser'
 
 module Fig; end
@@ -25,7 +24,7 @@ class Fig::Unparser::V0
     add_indent
 
     @text << %q<command ">
-    @text << statement.command
+    @text << statement.command.first.to_double_quotable_string
     @text << %Q<"\n>
 
     return
@@ -44,9 +43,9 @@ class Fig::Unparser::V0
     add_indent
 
     @text << 'retrieve '
-    @text << statement.var
+    @text << statement.variable
     @text << '->'
-    @text << statement.path
+    @text << statement.tokenized_path.to_double_quotable_string
     @text << "\n"
 
     return
@@ -80,7 +79,7 @@ class Fig::Unparser::V0
     @text << ' '
     @text << statement.name
     @text << '='
-    @text << statement.value
+    @text << statement.tokenized_value.to_double_quotable_string
     @text << "\n"
 
     return

data/lib/fig/unparser/v1.rb

@@ -22,10 +22,19 @@ class Fig::Unparser::V1
 
   def command(statement)
     add_indent
+    @text << %Q<command\n>
 
-    @text << %q<command ">
-    @text << statement.command
-    @text << %Q<"\n>
+    add_indent(@indent_level + 1)
+    statement.command.each do
+      |argument|
+
+      emit_tokenized_value argument
+      @text << ' '
+    end
+
+    @text << %Q<\n>
+    add_indent
+    @text << %Q<end\n>
 
     return
   end
@@ -38,6 +47,20 @@ class Fig::Unparser::V1
     return
   end
 
+  def retrieve(statement)
+    add_indent
+
+    @text << 'retrieve '
+    @text << statement.variable
+    @text << '->'
+
+    emit_tokenized_value statement.tokenized_path
+
+    @text << "\n"
+
+    return
+  end
+
   def grammar_description()
     return 'v1'
   end
@@ -61,17 +84,31 @@ class Fig::Unparser::V1
   end
 
   def environment_variable(statement, keyword)
-    # TODO: temporarily hack v0 grammar in here so we can test asset
-    # statements; proper implementation once asset statements are done.
     add_indent
 
     @text << keyword
     @text << ' '
     @text << statement.name
     @text << '='
-    @text << statement.value
+
+    emit_tokenized_value statement.tokenized_value
+
     @text << "\n"
 
     return
   end
+
+  def emit_tokenized_value(tokenized_value)
+    if tokenized_value.can_be_single_quoted?
+      @text << %q<'>
+      @text << tokenized_value.to_single_quoted_string
+      @text << %q<'>
+    else
+      @text << %q<">
+      @text << tokenized_value.to_escaped_string
+      @text << %q<">
+    end
+
+    return
+  end
 end