shellopts 2.0.0.pre.14 → 2.0.0

data/lib/shellopts/grammar.rb

@@ -0,0 +1,375 @@
+module ShellOpts
+  module Grammar
+    # Except for #parent, #children, and #token, all members are initialized by calling
+    # #parse on the object
+    #
+    class Node
+      attr_reader :parent
+      attr_reader :children
+      attr_reader :token
+
+      # Note that in derived classes 'super' should be called after member
+      # initialization because Node#initialize calls #attach on the parent that
+      # may need to access the members
+      def initialize(parent, token)
+        constrain parent, Node, nil
+        constrain parent, nil, lambda { |node| ALLOWED_PARENTS[self.class].any? { |klass| node.is_a?(klass) } }
+        constrain token, Token
+
+        @parent = parent
+        @children = []
+        @token = token
+        @parent.send(:attach, self) if @parent
+      end
+
+      def traverse(*klasses, &block)
+        do_traverse(Array(klasses).flatten, &block)
+      end
+
+      def parents() parent ? [parent] + parent.parents : [] end
+      def ancestors() parents.reverse end
+
+      def inspect
+        "#{self.class}"
+      end
+
+    protected
+      def attach(child)
+        @children << child
+      end
+
+      def do_traverse(klasses, &block)
+        yield(self) if klasses.empty? || klasses.any? { |klass| self.is_a?(klass) }
+        children.each { |node| node.traverse(klasses, &block) }
+      end
+    end
+
+    class IdrNode < Node
+      # Command of this object. This is different from #parent when a
+      # subcommand is nested on a higher level than its supercommand.
+      # Initialized by the analyzer
+      attr_reader :command
+
+      # Unique identifier of node (String) within the context of a program. nil
+      # for the Program object. It is the list of path elements concatenated
+      # with '.' and with internal '!' removed (eg. "cmd.opt" or "cmd.cmd!").
+      # Initialized by the parser
+      attr_reader :uid
+
+      # Path from Program object and down to this node. Array of identifiers.
+      # Empty for the Program object. Initialized by the parser
+      attr_reader :path
+
+      # Canonical identifier (Symbol) of the object
+      #
+      # For options, this is the canonical name of the objekt without the
+      # initial '-' or '--'. For commands it is the command name including the
+      # suffixed exclamation mark. Both options and commands have internal dashes
+      # replaced with underscores. Initialized by the parser
+      #
+      # Note that the analyzer fails with an an error if both --with-separator
+      # and --with_separator are used because they would both map to
+      # :with_separator
+      attr_reader :ident
+
+      # Canonical name (String) of the object
+      #
+      # This is the name of the object as the user sees it. For options it is
+      # the name of the first long option or the name of the first short option
+      # if there is no long option name. For commands it is the name without
+      # the exclamation mark. Initialized by the parser
+      attr_reader :name
+
+      # The associated attribute (Symbol) in the parent command object. nil if
+      # #ident is a reserved word. Initialized by the parser
+      attr_reader :attr
+
+    protected
+      def lookup(path)
+        path.empty? or raise ArgumentError, "Argument should be empty"
+        self
+      end
+    end
+
+    # Note that options are children of Command object but are attached to
+    # OptionGroup objects that in turn are attached to the command. This is
+    # done to be able to handle multiple options with common brief or
+    # descriptions
+    #
+    class Option < IdrNode
+      # Redefine command of this object
+      def command() parent.parent end # double-parent because options live in option groups
+
+      # Option group of this object
+      def group() parent end
+
+      # Short option identfiers
+      attr_reader :short_idents
+
+      # Long option identifiers
+      attr_reader :long_idents
+
+      # Short option names (including initial '-')
+      attr_reader :short_names
+
+      # Long option names (including initial '--')
+      attr_reader :long_names
+
+      # Identifiers of option. Include both short and long identifiers
+      def idents() short_idents + long_idents end
+
+      # Names of option. Includes both short and long option names
+      def names() short_names + long_names end # TODO: Should be in declaration order
+
+      # Name of argument or nil if not present
+      attr_reader :argument_name
+
+      # Type of argument (ArgType)
+      attr_reader :argument_type
+
+      # Enum values if argument type is an enumerator
+      def argument_enum() @argument_type.values end
+
+      def repeatable?() @repeatable end
+      def argument?() @argument end
+      def optional?() @optional end
+
+      def integer?() @argument_type.is_a? IntegerArgument end
+      def float?() @argument_type.is_a? FloatArgument end
+      def file?() @argument_type.is_a? FileArgument end
+      def enum?() @argument_type.is_a? EnumArgument end
+      def string?() argument? && !integer? && !float? && !file? && !enum? end
+
+      def match?(literal) argument_type.match?(literal) end
+
+      # Return true if the option can be assigned the given value
+#     def value?(value) ... end
+    end
+
+    # Note that all public attributes are assigned by #attach
+    #
+    class OptionGroup < Node
+      alias_method :command, :parent
+
+      # Array of options in declaration order
+      attr_reader :options
+
+      # Brief description of option(s)
+      attr_reader :brief
+
+      # Description of option(s)
+      attr_reader :description
+
+      def initialize(parent, token)
+        @options = []
+        @brief = nil
+        @default_brief = nil
+        @description = []
+        super(parent, token)
+      end
+
+    protected
+      def attach(child)
+        super
+        case child
+          when Option; @options << child
+          when Brief; @brief = child
+          when Paragraph; @description << child
+          when Code; @description << child
+        end
+      end
+    end
+
+    # Note that except for :options, all public attributes are assigned by
+    # #attach. :options and the member variables supporting the #[] and #[]=
+    # methods are initialized by the analyzer
+    #
+    class Command < IdrNode
+      # Supercommand or nil if this is the top-level Program object.
+      # Initialized by the analyzer
+      attr_reader :supercommand
+
+      # Brief description of command
+      attr_accessor :brief
+
+      # Description of command. Array of Paragraph or Code objects. Initialized
+      # by the parser
+      attr_reader :description
+
+      # Array of option groups in declaration order. Initialized by the parser
+      # TODO: Rename 'groups'
+      attr_reader :option_groups
+
+      # Array of options in declaration order. Initialized by the analyzer
+      attr_reader :options
+
+      # Array of sub-commands. Initialized by the parser but edited by the analyzer
+      attr_reader :commands
+
+      # Array of Arg objects. Initialized by the parser
+      attr_reader :specs
+
+      # Array of ArgDescr objects. Initialized by the parser
+      attr_reader :descrs
+
+      def initialize(parent, token)
+        @brief = nil
+        @default_brief = nil
+        @description = []
+        @option_groups = []
+        @options = []
+        @options_hash = {} # Initialized by the analyzer
+        @commands = []
+        @commands_hash = {} # Initialized by the analyzer
+        @specs = []
+        @descrs = []
+        super
+      end
+
+      # Maps from any (sub-)path, name or identifier of an option or command (including the
+      # suffixed '!') to the associated option. #[] and #key? can't be used
+      # until after the analyze phase
+      def [](key)
+        case key
+          when String; lookup(key.split("."))
+          when Symbol; lookup(key.to_s.sub(".", "!.").split(".").map(&:to_sym))
+          when Array; lookup(key)
+        else
+          nil
+        end
+      end
+
+      def key?(key) !self.[](key).nil? end
+
+      # Mostly for debug. Has questional semantics because it only lists local keys 
+      def keys() @options_hash.keys + @commands_hash.keys end
+
+      # Shorthand to get the associated Grammar::Command object from a Program
+      # or a Grammar::Command object
+      def self.command(obj)
+        constrain obj, Command, ::ShellOpts::Program
+        obj.is_a?(Command) ? obj : obj.__grammar__
+      end
+
+    protected
+      def attach(child)
+        super
+        case child
+          when OptionGroup; @option_groups << child
+          when Command; @commands << child
+          when ArgSpec; @specs << child
+          when ArgDescr; @descrs << child
+          when Brief; @brief = child
+          when Paragraph; @description << child
+          when Code; @description << child
+        end
+      end
+
+      def lookup(path)
+        key = path[0]
+        if path.size > 1
+          @commands_hash[key]&.lookup(path[1..-1])
+        elsif path.size == 1
+          @commands_hash[key] || @options_hash[key]
+        else
+          self
+        end
+      end
+    end
+
+    class Program < Command
+      # Lifted from .gemspec. TODO
+      attr_reader :info
+    end
+
+    class ArgSpec < Node
+      # List of Arg objects (initialized by the analyzer)
+      alias_method :command, :parent
+
+      attr_reader :arguments
+
+      def initialize(parent, token)
+        @arguments = []
+        super
+      end
+
+    protected
+      def attach(child)
+        arguments << child if child.is_a?(Arg)
+      end
+    end
+
+    class Arg < Node
+      alias_method :spec, :parent
+    end
+
+    # DocNode object has no children but lines. 
+    #
+    class DocNode < Node
+      # Array of :text tokens. Assigned by the parser
+      attr_reader :tokens
+
+      # The text of the node
+      def text() @text ||= tokens.map(&:source).join(" ") end
+
+      def lines() [text] end
+
+      def to_s() text end # FIXME
+
+      def initialize(parent, token, text = nil)
+        @tokens = [token]
+        @text = text
+        super(parent, token)
+      end
+    end
+
+    class ArgDescr < DocNode
+      alias_method :command, :parent
+    end
+
+    class Usage < ArgDescr
+    end
+
+    module WrappedNode
+      using Ext::Array::Wrap
+      def words() @words ||= text.split(" ") end
+      def lines(width, initial = 0) @lines ||= words.wrap(width, initial) end
+    end
+
+    class Brief < DocNode
+      include WrappedNode
+      alias_method :subject, :parent # Either a command or an option
+    end
+
+    class Paragraph < DocNode
+      include WrappedNode
+      alias_method :subject, :parent # Either a command or an option
+    end
+
+    class Code < DocNode
+      def text() @text ||= lines.join("\n") end
+      def lines() @lines ||= tokens.map { |t| " " * [t.charno - token.charno, 0].max + t.source } end
+    end
+
+    class Section < Node
+      def name() token.source end
+    end
+
+    class Node
+      ALLOWED_PARENTS = {
+        Program => [NilClass],
+        Command => [Command],
+        OptionGroup => [Command],
+        Option => [OptionGroup],
+        ArgSpec => [Command],
+        Arg => [ArgSpec],
+        ArgDescr => [Command],
+        Brief => [Command, OptionGroup, ArgSpec, ArgDescr],
+        Paragraph => [Command, OptionGroup],
+        Code => [Command, OptionGroup],
+        Section => [Program]
+      }
+    end
+  end
+end
+

data/lib/shellopts/interpreter.rb

@@ -0,0 +1,103 @@
+
+module ShellOpts
+  class Interpreter
+    attr_reader :expr
+    attr_reader :args
+
+    def initialize(grammar, argv, float: true, exception: false)
+      constrain grammar, Grammar::Program
+      constrain argv, [String]
+      @grammar, @argv = grammar, argv.dup
+      @float, @exception = float, exception
+    end
+
+    def interpret
+      @expr = command = Program.new(@grammar)
+      @seen = {} # Set of seen options by UID (using UID is needed when float is true)
+      @args = []
+
+      while arg = @argv.shift
+        if arg == "--"
+          break
+        elsif arg.start_with?("-")
+          interpret_option(command, arg)
+        elsif @args.empty? && subcommand_grammar = command.__grammar__[:"#{arg}!"]
+          command = Command.add_command(command, Command.new(subcommand_grammar))
+        else
+          if @float
+            @args << arg # This also signals that no more commands are accepted
+          else
+            @argv.unshift arg
+            break
+          end
+        end
+      end
+      [@expr, @args += @argv]
+    end
+
+    def self.interpret(grammar, argv, **opts)
+      self.new(grammar, argv, **opts).interpret
+    end
+
+  protected
+    # Lookup option in the command hierarchy and return pair of command and
+    # option associated command. Raise if not found
+    #
+    def find_option(command, name)
+      while command && (option = command.__grammar__[name]).nil?
+        command = command.__supercommand__
+      end
+      option or error "Unknown option '#{name}'"
+      [command, option]
+    end
+
+    def interpret_option(command, option)
+      # Split into name and argument
+      case option
+        when /^(--.+?)(?:=(.*))?$/
+          name, value, short = $1, $2, false
+        when /^(-.)(.+)?$/
+          name, value, short = $1, $2, true
+      end
+
+      option_command, option = find_option(command, name)
+      !@seen.key?(option.uid) || option.repeatable? or error "Duplicate option '#{name}'"
+      @seen[option.uid] = true
+
+      # Process argument
+      if option.argument?
+        if value.nil? && !option.optional?
+          if !@argv.empty?
+            value = @argv.shift
+          else
+            error "Missing argument for option '#{name}'"
+          end
+        end
+        value &&= interpret_option_value(option, name, value)
+      elsif value && short
+        @argv.unshift("-#{value}")
+        value = nil
+      elsif !value.nil?
+        error "No argument allowed for option '#{opt_name}'"
+      end
+      
+      Command.add_option(option_command, Option.new(option, name, value))
+    end
+
+    def interpret_option_value(option, name, value)
+      type = option.argument_type
+      if type.match?(name, value)
+        type.convert(value)
+      elsif value == ""
+        nil
+      else
+        error type.message
+      end
+    end
+
+    def error(msg)
+      raise Error, msg
+    end
+  end
+end
+

data/lib/shellopts/lexer.rb

@@ -0,0 +1,175 @@
+
+module ShellOpts
+  class Line
+    attr_reader :source
+    attr_reader :lineno 
+    attr_reader :charno
+    attr_reader :text
+
+    def initialize(lineno, charno, source)
+      @lineno, @source = lineno, source
+      @charno = charno + ((@source =~ /(\S.*?)\s*$/) || 0)
+      @text = $1 || ""
+    end
+
+    def blank?() @text == "" end
+
+    forward_to :@text, :=~, :!~
+
+    # Split on whitespace while keeping track of character position. Returns
+    # array of char, word tuples
+    def words
+      return @words if @words
+      @words = []
+      charno = self.charno
+      text.scan(/(\s*)(\S*)/)[0..-2].each { |spaces, word|
+        charno += spaces.size
+        @words << [charno, word] if word != ""
+        charno += word.size
+      }
+      @words
+    end
+
+    def to_s() text end
+    def dump() puts "#{lineno}:#{charno} #{text.inspect}" end
+  end
+
+  class Lexer
+    COMMAND_RE = /[a-z][a-z._-]*!/
+
+    DECL_RE = /^(?:-|--|\+|\+\+|(?:@(?:\s|$))|(?:[^\\!]\S*!(?:\s|$)))/
+
+    # Match ArgSpec argument words. TODO
+    SPEC_RE = /^[^a-z]{2,}$/
+
+    # Match ArgDescr words (should be at least two characters long)
+    DESCR_RE = /^[^a-z]{2,}$/
+
+    SECTIONS = %w(DESCRIPTION OPTIONS COMMANDS)
+
+    using Ext::Array::ShiftWhile
+
+    attr_reader :name # Name of program
+    attr_reader :source
+    attr_reader :tokens
+    
+    def oneline?() @oneline end
+
+    def initialize(name, source, oneline)
+      @name = name
+      @source = source
+      @oneline = oneline
+      @source += "\n" if @source[-1] != "\n" # Always terminate source with a newline
+    end
+
+    def lex(lineno = 1, charno = 1)
+      # Split source into lines and tag them with lineno and charno. Only the
+      # first line can have charno != 1
+      lines = source[0..-2].split("\n").map.with_index { |line,i|
+        l = Line.new(i + lineno, charno, line)
+        charno = 1
+        l
+      }
+
+      # Skip initial comments and blank lines and compute indent level
+      lines.shift_while { |line| line.text == "" || line.text.start_with?("#") && line.charno == 1 }
+      initial_indent = lines.first&.charno
+
+      # Create program token. The source is the program name
+      @tokens = [Token.new(:program, 0, 0, name)]
+
+      # Used to detect code blocks
+      last_nonblank = @tokens.first
+
+      # Process lines
+      while line = lines.shift
+        # Pass-trough blank lines
+        if line.to_s == ""
+          @tokens << Token.new(:blank, line.lineno, line.charno, "")
+          next
+        end
+          
+        # Ignore meta comments
+        if line.charno < initial_indent
+          next if line =~ /^#/
+          error_token = Token.new(:text, line.lineno, 0, "")
+          lexer_error error_token, "Illegal indentation"
+        end
+
+        # Line without escape sequences
+        source = line.text[(line.text =~ /^\\/ ? 1 : 0)..-1]
+
+        # Code lines
+        if last_nonblank.kind == :text && line.charno > last_nonblank.charno && line !~ DECL_RE
+          @tokens << Token.new(:text, line.lineno, line.charno, source)
+          lines.shift_while { |line| line.blank? || line.charno > last_nonblank.charno }.each { |line|
+            kind = (line.blank? ? :blank : :text)
+            @tokens << Token.new(kind, line.lineno, line.charno, line.text)
+          }
+
+        # Sections
+        elsif SECTIONS.include?(line.text)
+          @tokens << Token.new(:section, line.lineno, line.charno, line.text)
+
+        # Options, commands, usage, arguments, and briefs
+        elsif line =~ DECL_RE
+          words = line.words
+          while (charno, word = words.shift)
+            case word
+              when "@"
+                if words.empty?
+                  error_token = Token.new(:text, line.lineno, charno, "@")
+                  lexer_error error_token, "Empty '@' declaration"
+                end
+                source = words.shift_while { true }.map(&:last).join(" ")
+                @tokens << Token.new(:brief, line.lineno, charno, source)
+              when "--" # FIXME Rename argdescr
+                @tokens << Token.new(:usage, line.lineno, charno, "--")
+                source = words.shift_while { |_,w| w =~ DESCR_RE }.map(&:last).join(" ")
+                @tokens << Token.new(:usage_string, line.lineno, charno, source)
+              when "++" # FIXME Rename argspec
+                @tokens << Token.new(:spec, line.lineno, charno, "++")
+                words.shift_while { |c,w| 
+                  w =~ SPEC_RE and @tokens << Token.new(:argument, line.lineno, c, w) 
+                }
+              when /^-|\+/
+                @tokens << Token.new(:option, line.lineno, charno, word)
+              when /!$/
+                @tokens << Token.new(:command, line.lineno, charno, word)
+            else
+              source = [word, words.shift_while { |_,w| w !~ DECL_RE }.map(&:last)].flatten.join(" ")
+              @tokens << Token.new(:brief, line.lineno, charno, source)
+            end
+          end
+
+          # TODO: Move to parser and remove @oneline from Lexer
+          (token = @tokens.last).kind != :brief || !oneline? or 
+              lexer_error token, "Briefs are only allowed in multi-line specifications"
+
+        # Paragraph lines
+        else
+          @tokens << Token.new(:text, line.lineno, line.charno, source)
+        end
+        # FIXME Not sure about this
+#       last_nonblank = @tokens.last 
+        last_nonblank = @tokens.last if ![:blank, :usage_string, :argument].include? @tokens.last.kind 
+      end
+
+      # Move arguments and briefs before first command if one-line source
+#     if oneline? && cmd_index = @tokens.index { |token| token.kind == :command }
+#       @tokens = 
+#           @tokens[0...cmd_index] +
+#           @tokens[cmd_index..-1].partition { |token| ![:command, :option].include?(token.kind) }.flatten
+#     end
+
+      @tokens
+    end
+
+    def self.lex(name, source, oneline, lineno = 1, charno = 1)
+      Lexer.new(name, source, oneline).lex(lineno, charno)
+    end
+
+    def lexer_error(token, message) raise LexerError.new(token), message end
+  end
+end
+