ios_parser 0.5.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 34aee436a05fb00bbb2964694017e4592083d4411e93614be6767f8919a6b7c1
-  data.tar.gz: e7985ca414f954d59539c10abd1330c2842ac44f2f557aaac9a902145bfe2eec
+  metadata.gz: abbb84cb462b8506f141ce88c1333f8f25e8bc9f31e5bc166d2f76f275044529
+  data.tar.gz: 7ed4f70c4396caae3d84df7dc17176b5a4c1e1f4eede31cdf63af8d34c61285f
 SHA512:
-  metadata.gz: 54a4675f244006397240e590c277f84fcf626f8c561e5c49f5656d62c5bbce0dfda4bdd26c5fc5d8329da7fd8a697948dccbe58f2e1d507f56ccea6f450a032b
-  data.tar.gz: 583e81712114f810869f42c852211298e71adbb0cd51fbe1bf9ae5b9f964d31cbdcd70bd64212ddba808aca1383ce8302bff3adf5ef75f9716599ff0469fbec3
+  metadata.gz: 209623e73042e19dd5bb9108f3aee56ab357e2b84ad388f028e1055cf90eee73fc9040c448e6e394729880a3690abc38e2c945371dac73a7004db3bab78180a2
+  data.tar.gz: dc1b41a7d3e821d9bfdb2ce9e4b6cad8012396b2ba3ed1e39b56eecc7582870e829e8cb512ebca9a68ffb281c13aab1c5c7d6af48c53cec9ede3bbcc58078bb5

data/.travis.yml

@@ -14,4 +14,4 @@ matrix:
       env: JRUBY_OPTS='-Xcompat.version=2.0'
 bundler_args: --without guard
 before_install:
-  - if [ "jruby" != "$TRAVIS_RUBY_VERSION"  ]; then gem install bundler --without guard; fi
+  - if [ "jruby" != "$TRAVIS_RUBY_VERSION"  ]; then gem i rubygems-update -v '<3' && update_rubygems; gem install bundler -v 1.17.3 --without guard; fi

data/ext/ios_parser/c_lexer/lexer.c

@@ -2,6 +2,7 @@
 
 static VALUE rb_mIOSParser;
 static VALUE rb_cCLexer;
+static VALUE rb_cToken;
 VALUE rb_eLexError;
 
 typedef enum lex_token_state {
@@ -21,6 +22,9 @@ struct LexInfo {
     size_t pos;
     size_t token_start;
     size_t token_length;
+    size_t line;
+    size_t start_of_line;
+    size_t token_line;
     lex_token_state token_state;
     VALUE tokens;
     int indent;
@@ -46,13 +50,16 @@ typedef struct LexInfo LexInfo;
 
 #define CURRENT_CHAR(LEX) LEX->text[LEX->pos]
 #define TOKEN_EMPTY(LEX) LEX->token_length <= 0
+#define TOKEN_VALUE(TOK) RSTRUCT_GET(TOK, 0)
 
-#define MAKE_TOKEN(LEX, TOK) rb_ary_new3(2, rb_int_new(LEX->token_start), TOK)
-#define ADD_TOKEN(LEX, TOK) rb_ary_push(LEX->tokens, MAKE_TOKEN(LEX, TOK))
+#define ADD_TOKEN(LEX, TOK) rb_ary_push(LEX->tokens, make_token(LEX, TOK))
 
 #define CMD_LEN(CMD) (sizeof(CMD) - 1)
+
+static VALUE make_token(LexInfo *lex, VALUE tok);
+
 int is_certificate(LexInfo *lex) {
-    VALUE indent_ary, indent, command_ary, command;
+    VALUE indent_token, indent, command_token, command;
     int token_count, indent_pos, command_pos;
 
     token_count = RARRAY_LEN(lex->tokens);
@@ -62,16 +69,15 @@ int is_certificate(LexInfo *lex) {
     command_pos = token_count - 5;
     if (command_pos < 0) { return 0; }
 
-    indent_ary = rb_ary_entry(lex->tokens, indent_pos);
-    indent = rb_ary_entry(indent_ary, 1);
+    indent_token = rb_ary_entry(lex->tokens, indent_pos);
+    indent = TOKEN_VALUE(indent_token);
     if (TYPE(indent) != T_SYMBOL) { return 0; }
     if (rb_intern("INDENT") != SYM2ID(indent)) { return 0; }
 
-    command_ary = rb_ary_entry(lex->tokens, command_pos);
-    if (TYPE(command_ary) != T_ARRAY) { return 0; }
-    if (RARRAY_LEN(command_ary) < 2) { return 0; }
+    command_token = rb_ary_entry(lex->tokens, command_pos);
+    if (TYPE(command_token) != T_STRUCT) { return 0; }
 
-    command = rb_ary_entry(command_ary, 1);
+    command = TOKEN_VALUE(command_token);
     if (TYPE(command) != T_STRING) { return 0; }
 
     StringValue(command);
@@ -89,7 +95,7 @@ int is_banner_begin(LexInfo *lex) {
     if (banner_pos < 0) { return 0; }
 
     banner_ary = rb_ary_entry(lex->tokens, banner_pos);
-    banner = rb_ary_entry(banner_ary, 1);
+    banner = TOKEN_VALUE(banner_ary);
     if (TYPE(banner) != T_STRING) { return 0; }
 
     StringValue(banner);
@@ -148,6 +154,14 @@ static void deallocate(void * lex) {
     xfree(lex);
 }
 
+static VALUE make_token(LexInfo *lex, VALUE tok) {
+    return rb_struct_new(rb_cToken,
+                         tok,
+                         rb_int_new(lex->token_start),
+                         rb_int_new(lex->line),
+                         rb_int_new(lex->token_start - lex->start_of_line + 1));
+}
+
 static void mark(void *ptr) {
     LexInfo *lex = (LexInfo *)ptr;
     rb_gc_mark(lex->tokens);
@@ -164,6 +178,9 @@ static VALUE initialize(VALUE self, VALUE input_text) {
 
     lex->text = NULL;
     lex->pos = 0;
+    lex->line = 1;
+    lex->start_of_line = 0;
+    lex->token_line = 0;
     lex->token_start = 0;
     lex->token_length = 0;
     lex->token_state = LEX_STATE_ROOT;
@@ -180,6 +197,22 @@ static void process_root(LexInfo * lex);
 static void process_start_of_line(LexInfo * lex);
 static void start_banner(LexInfo * lex);
 
+static void find_start_of_line(LexInfo *lex, size_t from) {
+    size_t pos = from;
+
+    for (;;) {
+        if (IS_NEWLINE(lex->text[pos])) {
+            lex->start_of_line = pos + 1;
+            return;
+        } else if (pos <= 0) {
+            lex->start_of_line = 0;
+            return;
+        } else {
+            pos--;
+        }
+    }
+}
+
 static void process_newline(LexInfo *lex) {
     delimit(lex);
 
@@ -189,6 +222,8 @@ static void process_newline(LexInfo *lex) {
       lex->pos = lex->pos + 1;
       lex->token_start = lex->pos;
       lex->token_length = 0;
+      lex->token_line = 0;
+      lex->line = lex->line + 1;
       return;
     }
 
@@ -196,6 +231,7 @@ static void process_newline(LexInfo *lex) {
     ADD_TOKEN(lex, ID2SYM(rb_intern("EOL")));
     lex->token_state = LEX_STATE_INDENT;
     lex->indent = 0;
+    lex->line = lex->line + 1;
 }
 
 static void process_space(LexInfo *lex) {
@@ -204,11 +240,23 @@ static void process_space(LexInfo *lex) {
 
 static void process_comment(LexInfo *lex) {
     char c = CURRENT_CHAR(lex);
+    int token_count = RARRAY_LEN(lex->tokens);
+    VALUE last_token, last_value;
+
+    if (0 < token_count) {
+      last_token = rb_ary_entry(lex->tokens, token_count - 1);
+      last_value = TOKEN_VALUE(last_token);
+
+      if (TYPE(last_value) != T_SYMBOL) {
+        ADD_TOKEN(lex, ID2SYM(rb_intern("EOL")));
+      }
+    }
 
     if (IS_NEWLINE(c)) {
         delimit(lex);
         lex->token_state = LEX_STATE_INDENT;
         lex->indent = 0;
+        lex->line = lex->line + 1;
     }
 }
 
@@ -269,7 +317,7 @@ static void process_integer(LexInfo *lex) {
 }
 
 static void process_certificate(LexInfo *lex) {
-    char quit[5];
+    char quit[6] = "quit\n";
 
     strncpy(quit, &CURRENT_CHAR(lex) - 5, 5);
 
@@ -295,17 +343,31 @@ static void process_certificate(LexInfo *lex) {
         lex->token_length = 0;
 
         lex->token_start = lex->pos;
+        lex->line = lex->line + lex->token_line - 1;
+        find_start_of_line(lex, lex->pos);
         ADD_TOKEN(lex, ID2SYM(rb_intern("CERTIFICATE_END")));
 
-        process_newline(lex);
+        find_start_of_line(lex, lex->pos - 2);
+        lex->start_of_line++;
+        lex->token_start = lex->pos;
+        ADD_TOKEN(lex, ID2SYM(rb_intern("EOL")));
+
+        lex->token_state = LEX_STATE_INDENT;
+        lex->indent = 0;
+        lex->line = lex->line + 1;
+
         process_start_of_line(lex);
     } else {
+        if (IS_NEWLINE(CURRENT_CHAR(lex))) {
+            lex->token_line++;
+        }
         lex->token_length++;
     }
 }
 
 static void start_certificate(LexInfo *lex) {
     lex->indent_pos--;
+    lex->token_line = 0;
     rb_ary_pop(lex->tokens);
     rb_ary_pop(lex->tokens);
     ADD_TOKEN(lex, ID2SYM(rb_intern("CERTIFICATE_BEGIN")));
@@ -331,15 +393,22 @@ static void process_banner(LexInfo *lex) {
         lex->token_length++;
         delimit(lex);
         lex->token_start = lex->pos;
+        lex->line = lex->line + lex->token_line;
+        find_start_of_line(lex, lex->pos);
         ADD_TOKEN(lex, ID2SYM(rb_intern("BANNER_END")));
         if (lex->text[lex->pos + 1] == 'C') { lex->pos++; }
     } else if (!lex->banner_delimiter && is_banner_end_string(lex)) {
         lex->token_length -= 1;
         delimit(lex);
         lex->token_start = lex->pos;
+        lex->line = lex->line + lex->token_line;
+        find_start_of_line(lex, lex->pos);
         ADD_TOKEN(lex, ID2SYM(rb_intern("BANNER_END")));
     } else {
-        lex->token_length++;
+      if (IS_NEWLINE(lex->text[lex->pos + lex->token_length])) {
+        lex->token_line++;
+      }
+      lex->token_length++;
     }
 }
 
@@ -347,12 +416,17 @@ static void start_banner(LexInfo *lex) {
     char c = CURRENT_CHAR(lex);
     lex->banner_delimiter = (c == '\n') ? 0 : c;
     ADD_TOKEN(lex, ID2SYM(rb_intern("BANNER_BEGIN")));
+    if ('\n' == lex->text[lex->pos + 1]) lex->line++;
     if ('\n' == lex->text[lex->pos + 2]) lex->pos++;
 }
 
 static void process_start_of_line(LexInfo *lex) {
     char c = CURRENT_CHAR(lex);
 
+    if (lex->indent == 0) {
+        lex->start_of_line = lex->pos;
+    }
+
     if (IS_SPACE(c)) {
         lex->indent++;
         return;
@@ -487,7 +561,8 @@ static VALUE call(VALUE self, VALUE input_text) {
     }
 
     delimit(lex);
-    lex->token_start = lex->pos;
+    lex->token_start = lex->pos - 1;
+    lex->line = lex->line - 1;
 
     for (; lex->indent_pos > 0; lex->indent_pos--) {
         ADD_TOKEN(lex, ID2SYM(rb_intern("DEDENT")));
@@ -501,6 +576,7 @@ void Init_c_lexer() {
     rb_cCLexer = rb_define_class_under(rb_mIOSParser, "CLexer", rb_cObject);
     rb_eLexError = rb_define_class_under(rb_mIOSParser, "LexError",
                                          rb_eStandardError);
+    rb_cToken = rb_path2class("IOSParser::Token");
     rb_define_alloc_func(rb_cCLexer, allocate);
     rb_define_method(rb_cCLexer, "initialize", initialize, 0);
     rb_define_method(rb_cCLexer, "call", call, 1);

data/lib/ios_parser.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'ios_parser/token'
 
 module IOSParser
   class LexError < StandardError; end
@@ -6,11 +7,27 @@ module IOSParser
   def self.lexer
     if const_defined?(:PureLexer)
       PureLexer
+    else
+      c_lexer
+    end
+  rescue LoadError
+    pure_lexer
+  end
+
+  def self.c_lexer
+    if RUBY_VERSION < '2.1'
+      warn 'The C Lexer requires Ruby 2.1 or later. The pure Ruby lexer will '\
+           'be used instead. You can eliminate this warning by upgrading ruby '\
+           'or explicitly using the pure-Ruby lexer '\
+           "(require 'ios_parser/pure')"
+      pure_lexer
     else
       require_relative 'ios_parser/c_lexer'
       CLexer
     end
-  rescue LoadError
+  end
+
+  def self.pure_lexer
     require 'ios_parser/lexer'
     PureLexer
   end

data/lib/ios_parser/ios.rb

@@ -11,6 +11,7 @@ module IOSParser
       @document = Document.new(nil)
       @parent = parent
       @lexer  = lexer
+      @indent = 0
     end
 
     def tokens
@@ -29,40 +30,50 @@ module IOSParser
 
     def section(parent = nil)
       [].tap do |commands|
-        until tokens.empty? || tokens.first.last == :DEDENT
+        until tokens.empty? || tokens.first.value == :DEDENT
           commands.push(command(parent, @document))
         end
-        tokens.shift # discard :DEDENT
+        token = tokens.shift # discard :DEDENT
+        @indent -= 1 if token && token.value == :DEDENT
       end
     end
 
     def command(parent = nil, document = nil)
-      pos = tokens.first.first
-      opts = { args: arguments, parent: parent, document: document, pos: pos }
+      opts = {
+        tokens: command_tokens,
+        parent: parent,
+        document: document,
+        indent: @indent
+      }
 
       Command.new(opts).tap do |cmd|
         cmd.commands = subsections(cmd)
       end
     end
 
+    def command_tokens
+      toks = []
+      until tokens.empty? || tokens.first.value == :EOL
+        tok = tokens.shift
+        toks << tok unless argument_to_discard?(tok.value)
+      end
+      tokens.shift # discard :EOL
+      toks
+    end
+
+    def argument_to_discard?(arg)
+      arguments_to_discard.include?(arg)
+    end
+
     def arguments_to_discard
       [:INDENT, :DEDENT,
        :CERTIFICATE_BEGIN, :CERTIFICATE_END,
        :BANNER_BEGIN, :BANNER_END]
     end
 
-    def arguments
-      [].tap do |args|
-        until tokens.empty? || tokens.first.last == :EOL
-          _, arg = tokens.shift
-          args << arg unless arguments_to_discard.include?(arg)
-        end
-        tokens.shift # discard :EOL
-      end
-    end
-
     def subsections(parent = nil)
-      if !tokens.empty? && tokens.first.last == :INDENT
+      if !tokens.empty? && tokens.first.value == :INDENT
+        @indent += 1
         tokens.shift # discard :INDENT
         section(parent)
       else

data/lib/ios_parser/ios/command.rb

@@ -6,19 +6,22 @@ module IOSParser
     class Command
       include Enumerable
       include Queryable
-      attr_accessor :args, :commands, :parent, :pos, :document
-
-      def initialize(args: [], commands: [],
-                     parent: nil, pos: nil, document: nil)
-        @args = args
+      attr_accessor :commands, :parent, :document, :indent, :tokens
+      def initialize(tokens: [], commands: [],
+                     parent: nil, document: nil, indent: nil)
+        @tokens = tokens
         @commands = commands
         @parent = parent
-        @pos = pos
         @document = document
+        @indent = indent || 0
+      end
+
+      def args
+        tokens.map(&:value)
       end
 
       def name
-        args[0]
+        tokens.first.value
       end
 
       def ==(other)
@@ -37,6 +40,10 @@ module IOSParser
         parent ? parent.path + [parent.line] : []
       end
 
+      def pos
+        tokens.first && tokens.first.pos
+      end
+
       def indentation(base: 0)
         ' ' * (path.length - base)
       end
@@ -48,9 +55,10 @@ module IOSParser
 
       def inspect
         "<IOSParser::IOS::Command:0x#{object_id.to_s(16)} "\
-        "@args=#{args.inspect}, "\
+        "@tokens=#{tokens.inspect}, "\
         "@commands=#{commands.inspect}, "\
         "@pos=#{pos.inspect}, "\
+        "@indent=#{indent}, "\
         "@document=<IOSParser::IOS::Document:0x#{document.object_id.to_s(16)}>>"
       end
 
@@ -63,7 +71,8 @@ module IOSParser
         {
           args: args,
           commands: commands.map(&:to_hash),
-          pos: pos
+          pos: pos,
+          indent: indent
         }
       end
 

data/lib/ios_parser/lexer.rb

@@ -3,6 +3,7 @@ module IOSParser
     LexError = IOSParser::LexError
 
     attr_accessor :tokens, :token, :indents, :indent, :state, :char,
+                  :line, :start_of_line, :token_line,
                   :string_terminator
 
     def initialize
@@ -12,8 +13,10 @@ module IOSParser
       @indent  = 0
       @indents = [0]
       @state   = :root
-      @token_char = 0
-      @this_char  = -1
+      @this_char = -1
+      @line = 1
+      @start_of_line = 0
+      @token_line = 0
     end
 
     def call(input_text)
@@ -57,18 +60,37 @@ module IOSParser
       end
     end
 
-    def make_token(value, pos: nil)
+    def make_token(value, pos: nil, col: nil)
       pos ||= @token_start || @this_char
+      col ||= pos - start_of_line + 1
       @token_start = nil
-      [pos, value]
+      Token.new(value, pos, line, col)
+    end
+
+    def find_start_of_line(from: @this_char)
+      from.downto(0) do |pos|
+        if @text[pos] == "\n"
+          self.start_of_line = pos + 1
+          return start_of_line
+        end
+      end
+
+      self.line_start = 0
     end
 
     def comment
       self.state = :comment
-      return unless newline?
+      tokens << make_token(:EOL) if tokens.last &&
+                                    !tokens.last.value.is_a?(Symbol)
+      comment_newline if newline?
+    end
+
+    def comment_newline
       delimit
+      self.start_of_line = @this_char + 1
       self.state = :line_start
       self.indent = 0
+      self.line += 1
     end
 
     def comment?
@@ -81,16 +103,22 @@ module IOSParser
 
     def banner_begin
       self.state = :banner
+      self.token_line = 0
       tokens << make_token(:BANNER_BEGIN)
       @token_start = @this_char + 2
       @banner_delimiter = char == "\n" ? 'EOF' : char
+      return unless @text[@this_char + 1] == "\n"
+      self.token_line -= 1
+      self.line += 1
     end
 
     def banner_begin?
-      tokens[-2] && tokens[-2].last == 'banner'
+      tokens[-2] && tokens[-2].value == 'banner'
     end
 
     def banner
+      self.token_line += 1 if newline?
+
       if banner_end_char?
         banner_end_char
       elsif banner_end_string?
@@ -103,7 +131,10 @@ module IOSParser
     def banner_end_string
       self.state = :root
       token.chomp!(@banner_delimiter[0..-2])
-      tokens << make_token(token) << make_token(:BANNER_END)
+      tokens << make_token(token)
+      self.line += token_line
+      find_start_of_line
+      tokens << make_token(:BANNER_END)
       self.token = ''
     end
 
@@ -114,7 +145,10 @@ module IOSParser
     def banner_end_char
       self.state = :root
       banner_end_clean_token
-      tokens << make_token(token) << make_token(:BANNER_END)
+      tokens << make_token(token)
+      self.line += token_line
+      find_start_of_line
+      tokens << make_token(:BANNER_END)
       self.token = ''
     end
 
@@ -136,42 +170,60 @@ module IOSParser
     end
 
     def banner_garbage?(pos)
-      tokens[pos].last == :BANNER_END && tokens[pos + 1].last == 'C'
+      tokens[pos].value == :BANNER_END && tokens[pos + 1].value == 'C'
     end
 
     def certificate_begin?
-      tokens[-6] && tokens[-6].last == :INDENT &&
-        tokens[-5] && tokens[-5].last == 'certificate'
+      tokens[-6] && tokens[-6].value == :INDENT &&
+        tokens[-5] && tokens[-5].value == 'certificate'
     end
 
     def certificate_begin
       self.state = :certificate
       indents.pop
-      tokens[-2..-1] = [make_token(:CERTIFICATE_BEGIN, pos: tokens[-1][0])]
+      tokens[-2..-1] = [make_token(:CERTIFICATE_BEGIN, pos: tokens[-1].pos)]
+      self.token_line = 0
       certificate
     end
 
     def certificate
-      token[-5..-1] == "quit\n" ? certificate_end : token << char
+      if token.end_with?("quit\n")
+        certificate_end
+      else
+        self.token_line += 1 if char == "\n"
+        token << char
+      end
     end
 
     def certificate_end
       tokens.concat certificate_end_tokens
+      self.line += 1
       update_indentation
       @token_start = @this_char
 
       @token = ''
       self.state = :line_start
       self.indent = 0
+      self.line += 1
       root
     end
 
+    # rubocop: disable AbcSize
     def certificate_end_tokens
-      [
-        make_token(token[0..-6].gsub!(/\s+/, ' ').strip, pos: tokens[-1][0]),
-        make_token(:CERTIFICATE_END, pos: @this_char),
-        make_token(:EOL, pos: @this_char)
-      ]
+      cluster = []
+      cluster << make_token(certificate_token_value, pos: tokens[-1].pos)
+      self.line += self.token_line - 1
+      cluster << make_token(:CERTIFICATE_END, pos: @this_char, col: 1)
+      find_start_of_line(from: @this_char - 2)
+      cluster << make_token(:EOL,
+                            pos: @this_char,
+                            col: @this_char - start_of_line)
+      cluster
+    end
+    # rubocop: enable AbcSize
+
+    def certificate_token_value
+      token[0..-6].gsub!(/\s+/, ' ').strip
     end
 
     def integer
@@ -237,7 +289,7 @@ module IOSParser
 
     def space
       delimit
-      self.indent += 1 if tokens.last && tokens.last.last == :EOL
+      self.indent += 1 if tokens.last && tokens.last.value == :EOL
     end
 
     def space?
@@ -268,6 +320,8 @@ module IOSParser
       self.state = :line_start
       self.indent = 0
       tokens << make_token(:EOL)
+      self.start_of_line = @this_char + 1
+      self.line += 1
     end
 
     def newline?
@@ -304,7 +358,14 @@ module IOSParser
     end
 
     def pop_dedent
-      tokens << make_token(:DEDENT)
+      col =
+        if tokens.last.line == line
+          tokens.last.col
+        else
+          1
+        end
+
+      tokens << make_token(:DEDENT, col: col)
       indents.pop
     end
 
@@ -321,6 +382,7 @@ module IOSParser
       end
 
       delimit
+      self.line -= 1
       update_indentation
       scrub_banner_garbage
       tokens

data/lib/ios_parser/token.rb

@@ -0,0 +1,8 @@
+module IOSParser
+  Token = Struct.new(
+    :value,
+    :pos,
+    :line,
+    :col
+  )
+end

data/lib/ios_parser/version.rb

@@ -1,7 +1,7 @@
 module IOSParser
   class << self
     def version
-      '0.5.2'
+      '0.6.0'
     end
   end
 end

data/spec/lib/ios_parser/ios_spec.rb

@@ -25,22 +25,26 @@ module IOSParser
                     commands: [{ args: ['police', 300_000_000, 1_000_000,
                                         'exceed-action',
                                         'policed-dscp-transmit'],
-                                 commands: [{ args: %w[set dscp cs1],
-                                              commands: [], pos: 114 }],
-                                 pos: 50 }],
-                    pos: 24 },
+                                 commands: [
+                                   { args: %w[set dscp cs1],
+                                     commands: [], pos: 114, indent: 3 }
+                                 ],
+                                 pos: 50, indent: 2 }],
+                    pos: 24, indent: 1 },
 
                   { args: %w[class other_service],
                     commands: [{ args: ['police', 600_000_000, 1_000_000,
                                         'exceed-action',
                                         'policed-dscp-transmit'],
-                                 commands: [{ args: %w[set dscp cs2],
-                                              commands: [], pos: 214 },
-                                            { args: ['command_with_no_args'],
-                                              commands: [], pos: 230 }],
-                                 pos: 150 }],
-                    pos: 128 }],
-               pos: 0 }]
+                                 commands: [
+                                   { args: %w[set dscp cs2],
+                                     commands: [], pos: 214, indent: 3 },
+                                   { args: ['command_with_no_args'],
+                                     commands: [], pos: 230, indent: 3 }
+                                 ],
+                                 pos: 150, indent: 2 }],
+                    pos: 128, indent: 1 }],
+               pos: 0, indent: 0 }]
         }
       end
 
@@ -59,9 +63,9 @@ module IOSParser
         it('can be searched by an exact command') do
           expect(subject.find_all(name: 'set').map(&:to_hash))
             .to eq [{ args: %w[set dscp cs1],
-                      commands: [], pos: 114 },
+                      commands: [], pos: 114, indent: 3 },
                     { args: %w[set dscp cs2],
-                      commands: [], pos: 214 }]
+                      commands: [], pos: 214, indent: 3 }]
         end
 
         context 'can be searched by name and the first argument' do
@@ -92,8 +96,8 @@ module IOSParser
               [{ args: ['police', 300_000_000, 1_000_000, 'exceed-action',
                         'policed-dscp-transmit'],
                  commands: [{ args: %w[set dscp cs1],
-                              commands: [], pos: 114 }],
-                 pos: 50 }]
+                              commands: [], pos: 114, indent: 3 }],
+                 pos: 50, indent: 2 }]
             end
 
             context 'integer query' do
@@ -115,7 +119,7 @@ module IOSParser
                     .find('set')
                     .to_hash)
               .to eq(args: %w[set dscp cs1],
-                     commands: [], pos: 114)
+                     commands: [], pos: 114, indent: 3)
           end
         end # context 'nested search'
 
@@ -252,15 +256,15 @@ module IOSParser
         cmd_ary = [
           { args: ['ip', 'route', '10.0.0.1', '255.255.255.255',
                    'Null0'],
-            commands: [], pos: 0 },
+            commands: [], pos: 0, indent: 0 },
           { args: ['ip', 'route', '9.9.9.199', '255.255.255.255',
                    '42.42.42.142', 'name', 'PONIES'],
-            commands: [], pos: 40 },
+            commands: [], pos: 40, indent: 0 },
           { args: ['ip', 'route', 'vrf', 'Mgmt-intf', '0.0.0.0',
                    '0.0.0.0', '9.9.9.199'],
-            commands: [], pos: 100 },
+            commands: [], pos: 100, indent: 0 },
           { args: ['ip', 'route', '0.0.0.0/0', '11.11.0.111', 120],
-            commands: [], pos: 149 }
+            commands: [], pos: 149, indent: 0 }
         ]
 
         expect(result.find_all('ip route').map(&:to_hash)).to eq(cmd_ary)
@@ -269,7 +273,7 @@ module IOSParser
 
         cmd_hash = { args: ['ip', 'route', '9.9.9.199', '255.255.255.255',
                             '42.42.42.142', 'name', 'PONIES'],
-                     commands: [], pos: 40 }
+                     commands: [], pos: 40, indent: 0 }
         expect(result.find('ip route 9.9.9.199').to_hash).to eq(cmd_hash)
       end # end context '#call'
 

data/spec/lib/ios_parser/lexer_spec.rb

@@ -40,11 +40,11 @@ END
            'set', 'dscp', 'cs2', :EOL, :DEDENT, :DEDENT, :DEDENT]
         end
 
-        subject { klass.new.call(input).map(&:last) }
+        subject { klass.new.call(input).map(&:value) }
         it('enclosed in symbols') { should == output }
 
         it('enclosed in symbols (using the pure ruby lexer)') do
-          expect(subject_pure.map(&:last)).to eq output
+          expect(subject_pure.map(&:value)).to eq output
         end
       end
 
@@ -75,12 +75,12 @@ END
 
           it 'pure' do
             tokens = IOSParser::PureLexer.new.call(input)
-            expect(tokens.map(&:last)).to eq expectation
+            expect(tokens.map(&:value)).to eq expectation
           end # it 'pure' do
 
           it 'default' do
             tokens = IOSParser.lexer.new.call(input)
-            expect(tokens.map(&:last)).to eq expectation
+            expect(tokens.map(&:value)).to eq expectation
           end # it 'c' do
         end # context 'indented region' do
       end # context 'ASR indented regions' do
@@ -97,9 +97,11 @@ END
         end
 
         let(:output) do
-          [[0, 'banner'], [7, 'foobar'], [14, :BANNER_BEGIN],
-           [16, "asdf 1234 9786 asdf\nline 2\nline 3\n  "],
-           [52, :BANNER_END], [53, :EOL]]
+          [[0, 1, 1, 'banner'], [7, 1, 8, 'foobar'],
+           [14, 1, 15, :BANNER_BEGIN],
+           [16, 2, 17, "asdf 1234 9786 asdf\nline 2\nline 3\n  "],
+           [52, 5, 3, :BANNER_END], [53, 5, 4, :EOL]]
+            .map { |pos, line, col, val| Token.new(val, pos, line, col) }
         end
 
         it('tokenized and enclosed in symbols') { should == output }
@@ -119,8 +121,8 @@ END
           ['banner', 'exec', :BANNER_BEGIN, content, :BANNER_END, :EOL]
         end
 
-        it { expect(subject.map(&:last)).to eq output }
-        it { expect(subject_pure.map(&:last)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
       end
 
       context 'complex eos banner' do
@@ -131,14 +133,14 @@ END
           ['banner', 'motd', :BANNER_BEGIN, content, :BANNER_END, :EOL]
         end
 
-        it { expect(subject.map(&:last)).to eq output }
-        it { expect(subject_pure.map(&:last)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
       end
 
       context 'decimal number' do
         let(:input) { 'boson levels at 93.2' }
         let(:output) { ['boson', 'levels', 'at', 93.2] }
-        subject { klass.new.call(input).map(&:last) }
+        subject { klass.new.call(input).map(&:value) }
         it('converts to Float') { should == output }
       end
 
@@ -156,29 +158,33 @@ END
         end
 
         let(:output) do
-          [[0, 'crypto'],
-           [7, 'pki'],
-           [11, 'certificate'],
-           [23, 'chain'],
-           [29, 'TP-self-signed-0123456789'],
-           [54, :EOL],
-           [56, :INDENT],
-           [56, 'certificate'],
-           [68, 'self-signed'],
-           [80, '01'],
-           [85, :CERTIFICATE_BEGIN],
-           [85,
+          [[0, 1, 1, 'crypto'],
+           [7, 1, 8, 'pki'],
+           [11, 1, 12, 'certificate'],
+           [23, 1, 24, 'chain'],
+           [29, 1, 30, 'TP-self-signed-0123456789'],
+           [54, 1, 55, :EOL],
+           [56, 2, 2, :INDENT],
+           [56, 2, 2, 'certificate'],
+           [68, 2, 14, 'self-signed'],
+           [80, 2, 26, '01'],
+           [85, 3, 3, :CERTIFICATE_BEGIN],
+           [85, 3, 3,
             'FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF FFFFFFFF '\
             'FFFFFFFF EEEEEEEE EEEEEEEE EEEEEEEE EEEEEEEE EEEEEEEE EEEEEEEE '\
             'EEEEEEEE EEEEEEEE DDDDDDDD DDDDDDDD DDDDDDDD DDDDDDDD DDDDDDDD '\
             'DDDDDDDD DDDDDDDD DDDDDDDD AAAA'],
-           [323, :CERTIFICATE_END],
-           [323, :EOL],
-           [323, :DEDENT]]
+           [323, 6, 1, :CERTIFICATE_END],
+           [323, 6, 13, :EOL],
+           [323, 7, 1, :DEDENT]]
+            .map { |pos, line, col, val| Token.new(val, pos, line, col) }
         end
 
         subject { klass.new.call(input) }
-        it('tokenized') { expect(subject).to eq output }
+
+        it('tokenized') do
+          expect(subject).to eq output
+        end
 
         it('tokenized (using the pure ruby lexer)') do
           expect(subject_pure).to eq output
@@ -187,8 +193,8 @@ END
 
       context 'comments' do
         let(:input) { 'ip addr 127.0.0.0.1 ! asdfsdf' }
-        let(:output) { ['ip', 'addr', '127.0.0.0.1'] }
-        subject { klass.new.call(input).map(&:last) }
+        let(:output) { ['ip', 'addr', '127.0.0.0.1', :EOL] }
+        subject { klass.new.call(input).map(&:value) }
         it('dropped') { should == output }
       end
 
@@ -211,20 +217,20 @@ END
           ]
         end
 
-        it { expect(subject_pure.map(&:last)).to eq output }
-        it { expect(subject.map(&:last)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
       end # context 'quoted octothorpe' do
 
       context 'vlan range' do
         let(:input) { 'switchport trunk allowed vlan 50-90' }
         let(:output) do
           [
-            [0, 'switchport'],
-            [11, 'trunk'],
-            [17, 'allowed'],
-            [25, 'vlan'],
-            [30, '50-90']
-          ]
+            [0,  1, 1, 'switchport'],
+            [11, 1, 12, 'trunk'],
+            [17, 1, 18, 'allowed'],
+            [25, 1, 26, 'vlan'],
+            [30, 1, 31, '50-90']
+          ].map { |pos, line, col, val| Token.new(val, pos, line, col) }
         end
         it { should == output }
       end # context 'vlan range' do
@@ -247,31 +253,31 @@ END
           ]
         end
 
-        it { expect(subject_pure.map(&:last)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
       end
 
       context '# in the middle of a line is not a comment' do
         let(:input) { "vlan 1\n name #31337" }
         let(:output) { ['vlan', 1, :EOL, :INDENT, 'name', '#31337', :DEDENT] }
 
-        it { expect(subject_pure.map(&:last)).to eq output }
-        it { expect(subject.map(&:last)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
       end
 
       context '# at the start of a line is a comment' do
         let(:input) { "vlan 1\n# comment\nvlan 2" }
         let(:output) { ['vlan', 1, :EOL, 'vlan', 2] }
 
-        it { expect(subject_pure.map(&:last)).to eq output }
-        it { expect(subject.map(&:last)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
       end
 
       context '# after indentation is a comment' do
         let(:input) { "vlan 1\n # comment\nvlan 2" }
         let(:output) { ['vlan', 1, :EOL, :INDENT, :DEDENT, 'vlan', 2] }
 
-        it { expect(subject_pure.map(&:last)).to eq output }
-        it { expect(subject.map(&:last)).to eq output }
+        it { expect(subject_pure.map(&:value)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
       end
 
       context 'unterminated quoted string' do
@@ -297,23 +303,59 @@ END
         end
 
         let(:expected) do
+          expected_full.map(&:value)
+        end
+
+        let(:expected_full) do
           [
-            'router', 'static', :EOL,
-            :INDENT,
-            'address-family', 'ipv4', 'unicast', :EOL,
-            'address-family', 'ipv6', 'unicast', :EOL,
-            :DEDENT
-          ]
+            [0,  1, 1, 'router'],
+            [7,  1, 8, 'static'],
+            [13, 1, 14, :EOL],
+            [15, 2, 2, :INDENT],
+            [15, 2, 2, 'address-family'],
+            [30, 2, 17, 'ipv4'],
+            [35, 2, 22, 'unicast'],
+            [42, 2, 29, :EOL],
+            [47, 4, 2, 'address-family'],
+            [62, 4, 17, 'ipv6'],
+            [67, 4, 22, 'unicast'],
+            [74, 4, 29, :EOL],
+            [74, 4, 29, :DEDENT]
+          ].map { |pos, line, col, val| Token.new(val, pos, line, col) }
         end
 
         it 'lexes both subcommands' do
-          expect(subject.map(&:last)).to eq expected
+          expect(subject.map(&:value)).to eq expected
         end
 
         it 'lexes both subcommands (with the pure ruby lexer)' do
-          expect(subject_pure.map(&:last)).to eq expected
+          expect(subject_pure.map(&:value)).to eq expected
+        end
+
+        it 'lexes position, line, and column' do
+          expect(subject).to eq expected_full
+        end
+
+        it 'lexes position, line, and column (with the pure ruby lexer)' do
+          expect(subject_pure).to eq expected_full
         end
       end
+
+      context 'comment at end of line' do
+        let(:input) do
+          <<-END.unindent
+            description !
+            switchport access vlan 2
+          END
+        end
+
+        let(:output) do
+          ['description', :EOL, 'switchport', 'access', 'vlan', 2, :EOL]
+        end
+
+        it { expect(subject_pure.map(&:value)).to eq output }
+        it { expect(subject.map(&:value)).to eq output }
+      end # context 'comment at end of line' do
     end
   end
 end

data/spec/lib/ios_parser_spec.rb

@@ -24,22 +24,26 @@ describe IOSParser do
                     commands: [{ args: ['police', 300_000_000, 1_000_000,
                                         'exceed-action',
                                         'policed-dscp-transmit'],
-                                 commands: [{ args: %w[set dscp cs1],
-                                              commands: [], pos: 114 }],
-                                 pos: 50 }],
-                    pos: 24 },
+                                 commands: [
+                                   { args: %w[set dscp cs1],
+                                     commands: [], pos: 114, indent: 3 }
+                                 ],
+                                 pos: 50, indent: 2 }],
+                    pos: 24, indent: 1 },
 
                   { args: %w[class other_service],
                     commands: [{ args: ['police', 600_000_000, 1_000_000,
                                         'exceed-action',
                                         'policed-dscp-transmit'],
-                                 commands: [{ args: %w[set dscp cs2],
-                                              commands: [], pos: 214 },
-                                            { args: ['command_with_no_args'],
-                                              commands: [], pos: 230 }],
-                                 pos: 150 }],
-                    pos: 128 }],
-               pos: 0 }]
+                                 commands: [
+                                   { args: %w[set dscp cs2],
+                                     commands: [], pos: 214, indent: 3 },
+                                   { args: ['command_with_no_args'],
+                                     commands: [], pos: 230, indent: 3 }
+                                 ],
+                                 pos: 150, indent: 2 }],
+                    pos: 128, indent: 1 }],
+               pos: 0, indent: 0 }]
         }
       end
 
@@ -70,15 +74,18 @@ describe IOSParser do
                   {
                     args: %w[description blah blah blah],
                     commands: [],
-                    pos: 29
+                    pos: 29,
+                    indent: 1
                   },
                   {
                     args: ['match', 'access-group', 'fred'],
                     commands: [],
-                    pos: 57
+                    pos: 57,
+                    indent: 1
                   }
                 ],
-                pos: 0
+                pos: 0,
+                indent: 0
               }
             ]
         }
@@ -92,5 +99,22 @@ describe IOSParser do
         expect(actual).to eq(output)
       end
     end # context "partial outdent" do
+
+    context 'comment at end of line' do
+      let(:input) do
+        <<END.unindent
+          description !
+          switchport access vlan 2
+END
+      end
+
+      subject { described_class.parse(input) }
+
+      it 'parses both commands' do
+        should be_a IOSParser::IOS::Document
+        expect(subject.find(starts_with: 'description')).not_to be_nil
+        expect(subject.find(starts_with: 'switchport')).not_to be_nil
+      end
+    end
   end # describe '.parse'
 end # describe IOSParser

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ios_parser
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Ben Miller
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-16 00:00:00.000000000 Z
+date: 2019-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake-compiler
@@ -84,6 +84,7 @@ files:
 - lib/ios_parser/ios/queryable.rb
 - lib/ios_parser/lexer.rb
 - lib/ios_parser/pure.rb
+- lib/ios_parser/token.rb
 - lib/ios_parser/version.rb
 - spec/lib/ios_parser/ios/queryable_spec.rb
 - spec/lib/ios_parser/ios_spec.rb