bibtex-ruby 1.0.0

data/lib/bibtex/error.rb

@@ -0,0 +1,37 @@
+module BibTeX
+  
+  #
+  # Represents a lexical or syntactical error.
+  #
+  class Error < Element
+    
+    attr_reader :trace
+    
+    def initialize(trace=[])
+      @trace = trace
+    end
+
+    def trace=(trace)
+      raise(ArgumentError, "BibTeX::Error trace must be of type Array; was: #{trace.class.name}.") unless trace.kind_of?(Array)
+      @trace = trace
+    end
+    
+    def content
+      @trace.map { |e| e[1] }.join
+    end
+    
+    # Called when the element was added to a bibliography.
+    def added_to_bibliography(bibliography)
+      super(bibliography)
+      bibliography.errors << self
+      self
+    end
+    
+    # Called when the element was removed from a bibliography.
+    def removed_from_bibliography(bibliography)
+      super(bibliography)
+      bibliography.errors.delete(self)
+      self
+    end
+  end
+end

data/lib/bibtex/lexer.rb

@@ -0,0 +1,328 @@
+#--
+# BibTeX-Ruby
+# Copyright (C) 2010	Sylvester Keil <http://sylvester.keil.or.at>
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License
+# along with this program.	If not, see <http://www.gnu.org/licenses/>.
+#++
+
+require 'strscan'
+
+
+module BibTeX
+	
+	#
+	# The BibTeX::Lexer handles the lexical analysis of BibTeX bibliographies.
+	#
+	class Lexer
+
+		attr_reader :src, :options, :stack
+		
+		#
+		# Creates a new instance. Possible options and their respective
+		# default values are:
+		#
+		# - :include => [:errors] A list that may contain :meta_comments, and
+		#		:errors; depending on whether or not these are present, the respective
+		#		tokens are included in the parse tree.
+		# - :strict => true In strict mode objects can start anywhere; therefore
+		#		the `@' symbol is not possible except inside literals or @comment
+		#		objects; for a more lenient lexer set to false and objects are
+		#		expected to start after a new line (leading white space is permitted).
+		#
+		def initialize(options={})
+			@options = options
+			@options[:include] ||= [:errors]
+			@options[:strict] = true unless @options.has_key?(:strict)
+			@src = nil
+		end
+
+		# Sets the source for the lexical analysis and resets the internal state.
+		def src=(src)
+			@stack = []
+			@brace_level = 0
+			@mode = :meta
+			@active_object = nil
+			@src = StringScanner.new(src)
+			@line_breaks = []
+			@line_breaks << @src.pos until @src.scan_until(/\n|$/).empty?
+			@src.reset
+		end
+
+		# Returns the line number at a given position in the source.
+		def line_number_at(index)
+			(@line_breaks.find_index { |n| n >= index } || 0) + 1
+		end
+		
+		# Returns the next token from the parse stack.
+		def next_token
+			@stack.shift
+		end
+
+		def mode=(mode)
+			Log.debug("Lexer: switching to #{mode} mode...")
+
+			@active_object = case
+				when [:comment,:string,:preamble,:entry].include?(mode) then mode
+				when mode == :meta then nil
+				else @active_object
+			end
+
+			@mode = mode
+		end
+		
+		def mode
+			@mode
+		end
+
+		# Returns true if the lexer is currenty parsing a BibTeX object.
+		def bibtex_mode?
+			[:bibtex,:comment,:string,:preamble,:entry].include?(self.mode)
+		end
+		
+		# Returns true if the lexer is currently parsing meta comments.
+		def meta_mode?
+			self.mode == :meta
+		end
+
+		# Returns true if the lexer is currently parsing a braced-out expression.
+		def content_mode?
+			self.mode == :content
+		end
+
+		# Returns true if the lexer is currently parsing a string literal.
+		def literal_mode?
+			self.mode == :literal
+		end
+		
+		# Returns true if the lexer is currently parsing the given object type.
+		def is_active?(object)
+			@active_object == object
+		end
+		
+		# Pushes a value onto the parse stack.
+		def push(value)
+			case
+			when ([:CONTENT,:STRING_LITERAL].include?(value[0]) && value[0] == @stack.last[0])
+				@stack.last[1][0] << value[1]
+				@stack.last[1][1] = line_number_at(@src.pos)
+			when value[0] == :ERROR
+				@stack.push(value) if @options[:include].include?(:errors)
+				leave_object
+			when value[0] == :META_COMMENT
+				if @options[:include].include?(:meta_comments)
+					value[1] = [value[1], line_number_at(@src.pos)]
+					@stack.push(value)
+				end
+			else
+				value[1] = [value[1], line_number_at(@src.pos)]
+				@stack.push(value)
+			end
+			return self
+		end
+
+		# Start the lexical analysis.
+		def analyse(src=nil)
+			raise(ArgumentError, 'Lexer: failed to start analysis: no source given!') if src.nil? && @src.nil?
+			Log.debug('Lexer: starting lexical analysis...')
+			
+			self.src = src || @src.string
+			self.src.reset
+			
+			until self.src.eos?
+				case
+				when self.bibtex_mode?
+					parse_bibtex
+				when self.meta_mode?
+					parse_meta
+				when self.content_mode?
+					parse_content
+				when self.literal_mode?
+					parse_literal
+				end
+			end
+			
+			Log.debug('Lexer: finished lexical analysis.')
+			push [false, '$end']
+		end
+
+		def parse_bibtex
+			case
+			when self.src.scan(/[\t\r\n\s]+/o)
+			when self.src.scan(/\{/o)
+				@brace_level += 1
+				push [:LBRACE,'{']
+				if (@brace_level == 1 && is_active?(:comment)) || (@brace_level == 2 && is_active?(:entry))
+					self.mode = :content
+				end
+			when self.src.scan(/\}/o)
+				return error_unbalanced_braces if @brace_level < 1
+				@brace_level -= 1
+				push [:RBRACE,'}']
+				leave_object if @brace_level == 0
+			when self.src.scan( /=/o)
+				push [:EQ,'=']
+			when self.src.scan(/,/o)
+				push [:COMMA,',']
+			when self.src.scan(/#/o)
+				push [:SHARP,'#']
+			when self.src.scan(/\d+/o)
+				push [:NUMBER,self.src.matched]
+			when self.src.scan(/[a-z\d:_!$\.%&*-]+/io)
+				push [:NAME,self.src.matched]
+			when self.src.scan(/"/o)
+				self.mode = :literal
+			when self.src.scan(/@/o)
+				error_unexpected_token
+				enter_object
+			when self.src.scan(/./o)
+				error_unexpected_token
+				enter_object
+			end
+		end
+		
+		def parse_meta
+			match = self.src.scan_until(@options[:strict] ? /@[\t ]*/o : /(^|\n)[\t ]*@[\t ]*/o)
+			unless self.src.matched.nil?
+				push [:META_COMMENT, match.chop]
+				enter_object
+			else
+				push [:META_COMMENT,self.src.rest]
+				self.src.terminate
+			end
+		end
+
+		def parse_content
+			match = self.src.scan_until(/\{|\}/o)
+			case self.src.matched
+			when '{'
+				@brace_level += 1
+				push [:CONTENT,match]
+			when '}'
+				@brace_level -= 1
+				case
+				when @brace_level < 0
+					push [:CONTENT,match.chop]
+					error_unbalanced_braces
+				when @brace_level == 0
+					push [:CONTENT,match.chop]
+					push [:RBRACE,'}']
+					leave_object
+				when @brace_level == 1 && is_active?(:entry)
+					push [:CONTENT,match.chop]
+					push [:RBRACE,'}']
+					self.mode = :bibtex
+				else
+					push [:CONTENT, match]
+				end
+			else
+				push [:CONTENT,self.src.rest]
+				self.src.terminate
+				error_unterminated_content
+			end
+		end
+		
+		def parse_literal
+			match = self.src.scan_until(/[\{\}"\n]/o)
+			case self.src.matched
+			when '{'
+				@brace_level += 1
+				push [:STRING_LITERAL,match]
+			when '}'
+				@brace_level -= 1
+				if @brace_level < 1
+					push [:STRING_LITERAL,match.chop]
+					error_unbalanced_braces
+				else
+					push [:STRING_LITERAL,match]
+				end
+			when '"'
+				if @brace_level == 1
+					push [:STRING_LITERAL,match.chop]
+					self.mode = :bibtex
+				else
+					push [:STRING_LITERAL,match]
+				end
+			when "\n"
+				push [:STRING_LITERAL,match.chop]
+				error_unterminated_string
+			else
+				push [:STRING_LITERAL,self.src.rest]
+				self.src.terminate
+				error_unterminated_string
+			end
+		end
+		
+		# Called when the lexer encounters a new BibTeX object.
+		def enter_object
+			@brace_level = 0
+			self.mode = :bibtex
+			push [:AT,'@']
+
+			case
+			when self.src.scan(/string/io)
+				self.mode = :string
+				push [:STRING, self.src.matched]
+			when self.src.scan(/preamble/io)
+				self.mode = :preamble
+				push [:PREAMBLE, self.src.matched]
+			when self.src.scan(/comment/io)
+				self.mode = :comment
+				push [:COMMENT, self.src.matched]
+			when self.src.scan(/[a-z\d:_!\.$%&*-]+/io)
+				self.mode = :entry
+				push [:NAME, self.src.matched]
+			end
+		end
+
+		# Called when parser leaves a BibTeX object.
+		def leave_object
+			self.mode = :meta
+			@brace_level = 0
+		end
+
+
+		def error_unbalanced_braces
+			n = line_number_at(self.src.pos)
+			Log.warn("Lexer: unbalanced braces on line #{n}; brace level #{@brace_level}; mode #{@mode.inspect}.")
+			backtrace [:E_UNBALANCED_BRACES, [self.src.matched,n]]
+		end
+		
+		def error_unterminated_string
+			n = line_number_at(self.src.pos)
+			Log.warn("Lexer: unterminated string on line #{n}; brace level #{@brace_level}; mode #{@mode.inspect}.")
+			backtrace [:E_UNTERMINATED_STRING, [self.src.matched,n]]
+		end
+
+		def error_unterminated_content
+			n = line_number_at(self.src.pos)
+			Log.warn("Lexer: unterminated content on line #{n}; brace level #{@brace_level}; mode #{@mode.inspect}.")
+			backtrace [:E_UNTERMINATED_CONTENT, [self.src.matched,n]]
+		end
+		
+		def error_unexpected_token
+			n = line_number_at(self.src.pos)
+			Log.warn("Lexer: unexpected token `#{self.src.matched}' on line #{n}; brace level #{@brace_level}; mode #{@mode.inspect}.")
+			backtrace [:E_UNEXPECTED_TOKEN, [self.src.matched,n]]
+		end
+		
+		def backtrace(error)
+			trace = []
+			trace.unshift(@stack.pop) until @stack.empty? || (!trace.empty? && [:AT,:META_COMMENT].include?(trace[0][0]))
+			trace << error
+			push [:ERROR,trace]
+		end
+
+	end
+	
+end

data/lib/bibtex/parser.output

@@ -0,0 +1,578 @@
+
+
+-------- Grammar --------
+
+rule 1 bibliography: 
+rule 2 bibliography: objects
+rule 3 objects: object
+rule 4 objects: objects object
+rule 5 object: AT at_object
+rule 6 object: META_COMMENT
+rule 7 object: ERROR
+rule 8 at_object: comment
+rule 9 at_object: string
+rule 10 at_object: preamble
+rule 11 at_object: entry
+rule 12 comment: COMMENT LBRACE content RBRACE
+rule 13 content: 
+rule 14 content: CONTENT
+rule 15 preamble: PREAMBLE LBRACE string_value RBRACE
+rule 16 string: STRING LBRACE string_assignment RBRACE
+rule 17 string_assignment: NAME EQ string_value
+rule 18 string_value: string_literal
+rule 19 string_value: string_value SHARP string_literal
+rule 20 string_literal: NAME
+rule 21 string_literal: STRING_LITERAL
+rule 22 entry: entry_head assignments RBRACE
+rule 23 entry: entry_head assignments COMMA RBRACE
+rule 24 entry: entry_head RBRACE
+rule 25 entry_head: NAME LBRACE key COMMA
+rule 26 key: NAME
+rule 27 key: NUMBER
+rule 28 assignments: assignment
+rule 29 assignments: assignments COMMA assignment
+rule 30 assignment: NAME EQ value
+rule 31 value: string_value
+rule 32 value: NUMBER
+rule 33 value: LBRACE content RBRACE
+
+------- Symbols -------
+
+**Nonterminals, with rules where they appear
+
+  $start (17)
+    on right: 
+    on left : 
+  bibliography (18)
+    on right: 
+    on left : 1 2
+  objects (19)
+    on right: 2 4
+    on left : 3 4
+  object (20)
+    on right: 3 4
+    on left : 5 6 7
+  at_object (21)
+    on right: 5
+    on left : 8 9 10 11
+  comment (22)
+    on right: 8
+    on left : 12
+  string (23)
+    on right: 9
+    on left : 16
+  preamble (24)
+    on right: 10
+    on left : 15
+  entry (25)
+    on right: 11
+    on left : 22 23 24
+  content (26)
+    on right: 12 33
+    on left : 13 14
+  string_value (27)
+    on right: 15 17 19 31
+    on left : 18 19
+  string_assignment (28)
+    on right: 16
+    on left : 17
+  string_literal (29)
+    on right: 18 19
+    on left : 20 21
+  entry_head (30)
+    on right: 22 23 24
+    on left : 25
+  assignments (31)
+    on right: 22 23 29
+    on left : 28 29
+  key (32)
+    on right: 25
+    on left : 26 27
+  assignment (33)
+    on right: 28 29
+    on left : 30
+  value (34)
+    on right: 30
+    on left : 31 32 33
+
+**Terminals, with rules where they appear
+
+  $end (0) 
+  error (1) 
+  AT (2) 5
+  COMMA (3) 23 25 29
+  COMMENT (4) 12
+  CONTENT (5) 14
+  ERROR (6) 7
+  EQ (7) 17 30
+  LBRACE (8) 12 15 16 25 33
+  META_COMMENT (9) 6
+  NAME (10) 17 20 25 26 30
+  NUMBER (11) 27 32
+  PREAMBLE (12) 15
+  RBRACE (13) 12 15 16 22 23 24 33
+  SHARP (14) 19
+  STRING (15) 16
+  STRING_LITERAL (16) 21
+
+--------- State ---------
+
+state 0
+
+
+  AT            shift, and go to state 4
+  ERROR         shift, and go to state 6
+  META_COMMENT  shift, and go to state 5
+  $default      reduce using rule 1 (bibliography)
+
+  bibliography  go to state 1
+  objects       go to state 2
+  object        go to state 3
+
+state 1
+
+
+  $end          shift, and go to state 7
+
+
+state 2
+
+   2) bibliography : objects _
+   4) objects : objects _ object
+
+  AT            shift, and go to state 4
+  ERROR         shift, and go to state 6
+  META_COMMENT  shift, and go to state 5
+  $default      reduce using rule 2 (bibliography)
+
+  object        go to state 8
+
+state 3
+
+   3) objects : object _
+
+  $default      reduce using rule 3 (objects)
+
+
+state 4
+
+   5) object : AT _ at_object
+
+  COMMENT       shift, and go to state 14
+  NAME          shift, and go to state 18
+  PREAMBLE      shift, and go to state 15
+  STRING        shift, and go to state 16
+
+  at_object     go to state 9
+  comment       go to state 10
+  string        go to state 11
+  preamble      go to state 12
+  entry         go to state 13
+  entry_head    go to state 17
+
+state 5
+
+   6) object : META_COMMENT _
+
+  $default      reduce using rule 6 (object)
+
+
+state 6
+
+   7) object : ERROR _
+
+  $default      reduce using rule 7 (object)
+
+
+state 7
+
+
+  $end          shift, and go to state 19
+
+
+state 8
+
+   4) objects : objects object _
+
+  $default      reduce using rule 4 (objects)
+
+
+state 9
+
+   5) object : AT at_object _
+
+  $default      reduce using rule 5 (object)
+
+
+state 10
+
+   8) at_object : comment _
+
+  $default      reduce using rule 8 (at_object)
+
+
+state 11
+
+   9) at_object : string _
+
+  $default      reduce using rule 9 (at_object)
+
+
+state 12
+
+  10) at_object : preamble _
+
+  $default      reduce using rule 10 (at_object)
+
+
+state 13
+
+  11) at_object : entry _
+
+  $default      reduce using rule 11 (at_object)
+
+
+state 14
+
+  12) comment : COMMENT _ LBRACE content RBRACE
+
+  LBRACE        shift, and go to state 20
+
+
+state 15
+
+  15) preamble : PREAMBLE _ LBRACE string_value RBRACE
+
+  LBRACE        shift, and go to state 21
+
+
+state 16
+
+  16) string : STRING _ LBRACE string_assignment RBRACE
+
+  LBRACE        shift, and go to state 22
+
+
+state 17
+
+  22) entry : entry_head _ assignments RBRACE
+  23) entry : entry_head _ assignments COMMA RBRACE
+  24) entry : entry_head _ RBRACE
+
+  NAME          shift, and go to state 26
+  RBRACE        shift, and go to state 24
+
+  assignments   go to state 23
+  assignment    go to state 25
+
+state 18
+
+  25) entry_head : NAME _ LBRACE key COMMA
+
+  LBRACE        shift, and go to state 27
+
+
+state 19
+
+
+  $default      accept
+
+
+state 20
+
+  12) comment : COMMENT LBRACE _ content RBRACE
+
+  CONTENT       shift, and go to state 29
+  $default      reduce using rule 13 (content)
+
+  content       go to state 28
+
+state 21
+
+  15) preamble : PREAMBLE LBRACE _ string_value RBRACE
+
+  NAME          shift, and go to state 32
+  STRING_LITERAL  shift, and go to state 33
+
+  string_value  go to state 30
+  string_literal  go to state 31
+
+state 22
+
+  16) string : STRING LBRACE _ string_assignment RBRACE
+
+  NAME          shift, and go to state 35
+
+  string_assignment  go to state 34
+
+state 23
+
+  22) entry : entry_head assignments _ RBRACE
+  23) entry : entry_head assignments _ COMMA RBRACE
+  29) assignments : assignments _ COMMA assignment
+
+  COMMA         shift, and go to state 37
+  RBRACE        shift, and go to state 36
+
+
+state 24
+
+  24) entry : entry_head RBRACE _
+
+  $default      reduce using rule 24 (entry)
+
+
+state 25
+
+  28) assignments : assignment _
+
+  $default      reduce using rule 28 (assignments)
+
+
+state 26
+
+  30) assignment : NAME _ EQ value
+
+  EQ            shift, and go to state 38
+
+
+state 27
+
+  25) entry_head : NAME LBRACE _ key COMMA
+
+  NAME          shift, and go to state 40
+  NUMBER        shift, and go to state 41
+
+  key           go to state 39
+
+state 28
+
+  12) comment : COMMENT LBRACE content _ RBRACE
+
+  RBRACE        shift, and go to state 42
+
+
+state 29
+
+  14) content : CONTENT _
+
+  $default      reduce using rule 14 (content)
+
+
+state 30
+
+  15) preamble : PREAMBLE LBRACE string_value _ RBRACE
+  19) string_value : string_value _ SHARP string_literal
+
+  RBRACE        shift, and go to state 43
+  SHARP         shift, and go to state 44
+
+
+state 31
+
+  18) string_value : string_literal _
+
+  $default      reduce using rule 18 (string_value)
+
+
+state 32
+
+  20) string_literal : NAME _
+
+  $default      reduce using rule 20 (string_literal)
+
+
+state 33
+
+  21) string_literal : STRING_LITERAL _
+
+  $default      reduce using rule 21 (string_literal)
+
+
+state 34
+
+  16) string : STRING LBRACE string_assignment _ RBRACE
+
+  RBRACE        shift, and go to state 45
+
+
+state 35
+
+  17) string_assignment : NAME _ EQ string_value
+
+  EQ            shift, and go to state 46
+
+
+state 36
+
+  22) entry : entry_head assignments RBRACE _
+
+  $default      reduce using rule 22 (entry)
+
+
+state 37
+
+  23) entry : entry_head assignments COMMA _ RBRACE
+  29) assignments : assignments COMMA _ assignment
+
+  NAME          shift, and go to state 26
+  RBRACE        shift, and go to state 47
+
+  assignment    go to state 48
+
+state 38
+
+  30) assignment : NAME EQ _ value
+
+  LBRACE        shift, and go to state 52
+  NAME          shift, and go to state 32
+  NUMBER        shift, and go to state 51
+  STRING_LITERAL  shift, and go to state 33
+
+  string_literal  go to state 31
+  string_value  go to state 49
+  value         go to state 50
+
+state 39
+
+  25) entry_head : NAME LBRACE key _ COMMA
+
+  COMMA         shift, and go to state 53
+
+
+state 40
+
+  26) key : NAME _
+
+  $default      reduce using rule 26 (key)
+
+
+state 41
+
+  27) key : NUMBER _
+
+  $default      reduce using rule 27 (key)
+
+
+state 42
+
+  12) comment : COMMENT LBRACE content RBRACE _
+
+  $default      reduce using rule 12 (comment)
+
+
+state 43
+
+  15) preamble : PREAMBLE LBRACE string_value RBRACE _
+
+  $default      reduce using rule 15 (preamble)
+
+
+state 44
+
+  19) string_value : string_value SHARP _ string_literal
+
+  NAME          shift, and go to state 32
+  STRING_LITERAL  shift, and go to state 33
+
+  string_literal  go to state 54
+
+state 45
+
+  16) string : STRING LBRACE string_assignment RBRACE _
+
+  $default      reduce using rule 16 (string)
+
+
+state 46
+
+  17) string_assignment : NAME EQ _ string_value
+
+  NAME          shift, and go to state 32
+  STRING_LITERAL  shift, and go to state 33
+
+  string_value  go to state 55
+  string_literal  go to state 31
+
+state 47
+
+  23) entry : entry_head assignments COMMA RBRACE _
+
+  $default      reduce using rule 23 (entry)
+
+
+state 48
+
+  29) assignments : assignments COMMA assignment _
+
+  $default      reduce using rule 29 (assignments)
+
+
+state 49
+
+  19) string_value : string_value _ SHARP string_literal
+  31) value : string_value _
+
+  SHARP         shift, and go to state 44
+  $default      reduce using rule 31 (value)
+
+
+state 50
+
+  30) assignment : NAME EQ value _
+
+  $default      reduce using rule 30 (assignment)
+
+
+state 51
+
+  32) value : NUMBER _
+
+  $default      reduce using rule 32 (value)
+
+
+state 52
+
+  33) value : LBRACE _ content RBRACE
+
+  CONTENT       shift, and go to state 29
+  $default      reduce using rule 13 (content)
+
+  content       go to state 56
+
+state 53
+
+  25) entry_head : NAME LBRACE key COMMA _
+
+  $default      reduce using rule 25 (entry_head)
+
+
+state 54
+
+  19) string_value : string_value SHARP string_literal _
+
+  $default      reduce using rule 19 (string_value)
+
+
+state 55
+
+  17) string_assignment : NAME EQ string_value _
+  19) string_value : string_value _ SHARP string_literal
+
+  SHARP         shift, and go to state 44
+  $default      reduce using rule 17 (string_assignment)
+
+
+state 56
+
+  33) value : LBRACE content _ RBRACE
+
+  RBRACE        shift, and go to state 57
+
+
+state 57
+
+  33) value : LBRACE content RBRACE _
+
+  $default      reduce using rule 33 (value)
+