rley 0.2.08 → 0.2.09

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZmY0MDhhM2RlNzcxYzk3ZDJkMGM5YWVjY2Q0NWRlNGZhZmYyMjdmOQ==
+    MTgzNDQzMWNlYTAwMWNiZjIxZjYxYzBiY2YyZjYyMDE2MjA4NjJkZg==
   data.tar.gz: !binary |-
-    NGNmZDI2MmYyNDBhZWZiNzdhZDhjNmNjMjQxMDRkMDI0NTZkMDA3YQ==
+    YTFjMDBmNWYzYmUwNjI4ZDNiNjczMjNhMTlhZjRjYzhlODUwMWM5Nw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    YTliNWVmM2NkN2ZjZjkyZjJhZGVhODE3M2M5MGQ5ZGI5NWM1MTI1ZGJjZWVl
-    ZTg4ZTFhODNjNzA3MmZlMmY4YTRjNDc0YzQ0M2FlNzE2NTU0YjQzMDBjYjUy
-    MmZjY2E5NzcyYzFiNTA2ZTg3Yjc1ZmUwNGJlNjUwOWM1YzY3ZWY=
+    MjAwZjQ4YzdmMTMxOTAxY2IxMTQ0N2QzYzI0MWM1ZWIyOGJmNGZjMWE0YzVk
+    MTZlOGZmZWU2YzhiZmFhN2I5YzU5NGFkNDk0MTY2NjFmNzk4MzBhZWM5NjUz
+    NDgyODQxNTkzMWFkMGM2OTJhMDYyYWMxMjY1ZjQ0NjZlMjdhNTU=
   data.tar.gz: !binary |-
-    ZTM0M2IwODkwNjJlMDZkYzQzNTM4YmRkNDY5ZDI1YjQ1M2E5Y2IzZTA1OWIz
-    MGM4MDAxZTNlNDM2N2Q1ZTI3NzcxOGU0NTBmMGFmYTU2MDBhNDMxNjdlNzI5
-    ZTRlYTJkMzczNWZkZjk1NjkxZTVmYzc3N2ZjY2FlMDVhMTAyYjg=
+    ZmZlMGNlY2Q1M2E1MzkwMThlNmI0NTEwOTEzNTcxNzZjYWU2MGE0MDQ3ODUz
+    MjhjYjZjMjQxMGQzZjJkMTYwZDNhNWViMjhmZDk4ZGM1MjFkYjIxNTJkYjBm
+    MjZkMTE2YzdhYjg0NTRiMzhkZjMzOWVlY2VlMGFlMWU4ZDAxOGM=

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+### 0.2.08 / 2015-04-28
+* [NEW] Added folder with JSON demo parser under `examples\parsers\demo-JSON`.
+
 ### 0.2.07 / 2015-04-22
 * [NEW] Rake file added in `examples` folder. It allows to run all the examples at once.
 

data/README.md

@@ -40,6 +40,22 @@ This project is in "earley" stage.
 - Provide documentation and examples
 
 
+### Other similar Ruby projects ###
+__Rley__ isn't the sole Ruby implementation of the Earley parser algorithm.  
+Here are a few other ones:  
+- [Kanocc gem](https://rubygems.org/gems/kanocc) -- Advertised as a Ruby based parsing and translation framework.  
+  Although the gem dates from 2009, the author still maintains its in a public repository in [Github](https://github.com/surlykke/Kanocc)  
+  The grammar symbols (tokens and non-terminals) must be represented as (sub)classes.
+  Grammar rules are methods of the non-terminal classes. A rule can have a block code argument
+  that specifies the semantic action when that rule is applied.  
+- [lc1 project](https://github.com/kp0v/lc1) -- Advertised as a combination of Earley and Viterbi algorithms for [Probabilistic] Context Free Grammars   
+  Aimed in parsing brazilian portuguese.  
+  [earley project](https://github.com/joshingly/earley) -- An Earley parser (grammar rules are specified in JSON format).  
+  The code doesn't seem to be maintained: latest commit dates from Nov. 2011.  
+- [linguist project](https://github.com/davidkellis/linguist) -- Advertised as library for parsing context-free languages.  
+  It is a recognizer not a parser. In other words it can only tell whether a given input 
+  conforms to the grammar rules or not. As such it cannot build parse trees.  
+  The code doesn't seem to be maintained: latest commit dates from Oct. 2011.
 
 Copyright
 ---------

data/examples/parsers/demo-JSON/JSON_grammar.rb

@@ -0,0 +1,31 @@
+# Purpose: to demonstrate how to build and render a parse tree for JSON
+# language
+require 'rley'  # Load the gem
+
+
+########################################
+# Define a grammar for JSON
+builder = Rley::Syntax::GrammarBuilder.new
+builder.add_terminals('KEYWORD') # For true, false, null keywords
+builder.add_terminals('JSON_STRING', 'JSON_NUMBER')
+builder.add_terminals('LACCOL', 'RACCOL') # For '{', '}' delimiters
+builder.add_terminals('LBRACKET', 'RBRACKET') # For '[', ']' delimiters
+builder.add_terminals('COLON', 'COMMA') # For ':', ',' separators
+builder.add_production('json_text' => 'json_value')
+builder.add_production('json_value' => 'json_object')
+builder.add_production('json_value' => 'json_array')
+builder.add_production('json_value' => 'JSON_STRING')
+builder.add_production('json_value' => 'JSON_NUMBER')
+builder.add_production('json_value' => 'KEYWORD')
+builder.add_production('json_object' => %w[LACCOL json_pairs RACCOL])
+builder.add_production('json_object' => ['LACCOL', 'RACCOL'])
+builder.add_production('json_pairs' => %w[json_pairs COMMA single_pair])
+builder.add_production('json_pairs' => 'single_pair')
+builder.add_production('single_pair' => %w[JSON_STRING COLON json_value])
+builder.add_production('json_array' => %w[LBRACKET array_items RBRACKET])
+builder.add_production('json_array' => ['RBRACKET', 'RBRACKET'])
+builder.add_production('array_items' => %w[array_items COMMA json_value])
+builder.add_production('array_items' => %w[json_value])
+
+# And now build the grammar...
+GrammarJSON = builder.grammar

data/examples/parsers/demo-JSON/JSON_lexer.rb

@@ -0,0 +1,114 @@
+# File: JSON_lexer.rb
+# Lexer for the JSON data format
+require 'rley'  # Load the gem
+require 'strscan'
+
+# Lexer for JSON.
+class JSONLexer
+  attr_reader(:scanner)
+  attr_reader(:lineno)
+  attr_reader(:line_start)
+  attr_reader(:name2symbol)
+
+  @@lexeme2name = {
+    '{' => 'LACCOL',
+    '}' => 'RACCOL',
+    '[' => 'LBRACKET',
+    ']' => 'RBRACKET',
+    ',' => 'COMMA',
+    ':' => 'COLON'
+  }
+
+  class ScanError < StandardError ; end
+
+public
+  def initialize(source, aGrammar)
+    @scanner = StringScanner.new(source)
+    @name2symbol = aGrammar.name2symbol
+    @lineno =  1
+  end
+
+  def tokens()
+    tok_sequence = []
+    until @scanner.eos? do
+      token = _next_token
+      tok_sequence << token unless token.nil?
+    end
+
+    return tok_sequence
+  end
+
+private
+  def _next_token()
+    token = nil
+    skip_whitespaces
+    curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
+
+    begin
+      break if curr_ch.nil?
+
+      case curr_ch
+        when '{', '}', '[', ']', ',', ':'
+          type_name = @@lexeme2name[curr_ch]
+          token_type = name2symbol[type_name]
+          token = Rley::Parser::Token.new(curr_ch, token_type)
+
+        # LITERALS
+        when '"'  # Start string delimiter found
+          value = scanner.scan(/([^"\\]|\\.)*/)
+          end_delimiter = scanner.getch()
+          raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
+          token_type = name2symbol['JSON_STRING']
+          token = Rley::Parser::Token.new(value, token_type)
+
+        when /[ftn]/  # First letter of keywords
+          @scanner.pos = scanner.pos - 1 # Simulate putback
+          keyw = scanner.scan(/false|true|null/)
+          if keyw.nil?
+            invalid_keyw = scanner.scan(/\w+/)
+            raise ScanError.new("Invalid keyword: #{invalid_keyw}")
+          else
+            token_type = name2symbol['KEYWORD']
+            token = Rley::Parser::Token.new(keyw, token_type)
+          end
+
+
+        when /[-0-9]/ # Start character of number literal found
+          @scanner.pos = scanner.pos - 1 # Simulate putback
+          value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
+          token_type = name2symbol['JSON_NUMBER']
+          token = Rley::Parser::Token.new(value, token_type)
+
+
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : curr_ch
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError.new("Unknown token #{erroneous}")
+      end #case
+
+
+    end while (token.nil? && curr_ch = scanner.getch())
+
+    return token
+  end
+
+
+  def skip_whitespaces()
+    matched = scanner.scan(/[ \t\f\n\r]+/)
+    return if matched.nil?
+
+    newline_count = 0
+    matched.scan(/\n\r?|\r/) { |_| newline_count += 1 }
+    newline_detected(newline_count)
+  end
+
+
+  def newline_detected(count)
+    @lineno += count
+    @line_start = scanner.pos()
+  end
+
+end # class
+
+

data/examples/parsers/demo-JSON/JSON_parser.rb

@@ -0,0 +1,89 @@
+# Purpose: to demonstrate how to build and render a parse tree for JSON
+# language
+require 'pp'
+require 'rley'  # Load the gem
+require_relative 'json_lexer'
+
+# Steps to render a parse tree (of a valid parsed input):
+# 1. Define a grammar
+# 2. Create a parser for that grammar
+# 3. Tokenize the input
+# 4. Let the parser process the input
+# 5. Generate a parse tree from the parse result
+# 6. Render the parse tree (in JSON)
+
+########################################
+# Step 1. Load a grammar for JSON
+require_relative 'JSON_grammar'
+
+# A JSON parser derived from our general Earley parser.
+class JSONParser < Rley::Parser::EarleyParser
+  attr_reader(:source_file)
+
+  # Constructor
+  def initialize()
+    # Builder the Earley parser with the JSON grammar
+    super(GrammarJSON)
+  end
+  
+  def parse_file(aFilename)
+    tokens = tokenize_file(aFilename)
+    result = parse(tokens)
+
+    return result
+  end
+  
+  private
+  
+  def tokenize_file(aFilename)
+    input_source = nil
+    File.open(aFilename, 'r') { |f| input_source = f.read }
+
+    lexer = JSONLexer.new(input_source, GrammarJSON)
+    return lexer.tokens
+  end
+end # class
+
+=begin
+########################################
+# Step 3. Create a parser for that grammar
+# parser = Rley::Parser::EarleyParser.new(GrammarJSON)
+parser = JSONParser.new
+
+
+########################################
+# Step 4. Tokenize the input file
+file_name = 'sample02.json'
+=begin
+input_source = nil
+File.open(file_name, 'r') { |f| input_source = f.read }
+
+lexer = JSONLexer.new(input_source, GrammarJSON)
+tokens =  lexer.tokens
+#=end
+
+########################################
+# Step 5. Let the parser process the input
+result = parser.parse_file(file_name) # parser.parse(tokens)
+unless result.success?
+  puts "Parsing of '#{file_name}' failed"
+  exit(1)
+end
+
+########################################
+# Step 6. Generate a parse tree from the parse result
+ptree = result.parse_tree
+
+########################################
+# Step 7. Render the parse tree (in JSON)
+# Let's create a parse tree visitor
+visitor = Rley::ParseTreeVisitor.new(ptree)
+
+#Here we create a renderer object...
+renderer = Rley::Formatter::Json.new(STDOUT)
+
+# Now emit the parse tree as JSON on the console output
+puts "JSON rendering of the parse tree for '#{file_name}' input:"
+renderer.render(visitor)
+=end
+# End of file

data/examples/parsers/demo-JSON/demo_json.rb

@@ -0,0 +1,42 @@
+require_relative 'JSON_parser'
+
+# Create a JSON parser object
+parser = JSONParser.new 
+
+# Parse the input file with name given in command-line
+if ARGV.empty?
+  msg = <<-END_MSG
+Command-line symtax:
+  ruby #{__FILE__} filename
+  where:
+    filename is the name of a JSON file
+    
+  Example:
+  ruby #{__FILE__} sample01.json
+END_MSG
+  puts msg
+  exit(1)
+end
+file_name = ARGV[0]
+result = parser.parse_file(file_name) # result object contains parse details
+
+unless result.success?
+  # Stop if the parse failed...
+  puts "Parsing of '#{file_name}' failed"
+  exit(1)
+end
+
+# Generate a parse tree from the parse result
+ptree = result.parse_tree
+
+# Do something with the parse tree: render it on the output console.
+# Step a: Let's create a parse tree visitor
+visitor = Rley::ParseTreeVisitor.new(ptree)
+
+# Step b: Select the rendition format to be JSON
+renderer = Rley::Formatter::Json.new(STDOUT)
+
+# Step c: Now emit the parse tree as JSON on the console output
+puts "JSON rendering of the parse tree for '#{file_name}' input:"
+renderer.render(visitor)
+# End of file

data/examples/parsers/parsing_L1.rb

@@ -102,10 +102,6 @@ def tokenizer(aText, aGrammar)
   return tokens
 end
 
-########################################
-# Step 3. Create a parser for that grammar
-parser = Rley::Parser::EarleyParser.new(grammar_l1)
-
 ########################################
 # Step 3. Tokenize the input
 valid_input = 'I want the flight from Alaska through Chicago to Houston'
@@ -113,6 +109,10 @@ valid_input = 'I want the flight from Alaska through Chicago to Houston'
 # Another sentence: it is a flight from Chicago
 tokens = tokenizer(valid_input, grammar_l1)
 
+########################################
+# Step 4. Create a parser for that grammar
+parser = Rley::Parser::EarleyParser.new(grammar_l1)
+
 ########################################
 # Step 5. Let the parser process the input
 result = parser.parse(tokens)

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.2.08'
+  Version = '0.2.09'
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.2.08
+  version: 0.2.09
 platform: ruby
 authors:
 - Dimitri Geshef
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-22 00:00:00.000000000 Z
+date: 2015-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -101,6 +101,10 @@ files:
 - README.md
 - examples/grammars/grammar_abc.rb
 - examples/grammars/grammar_L0.rb
+- examples/parsers/demo-JSON/demo_json.rb
+- examples/parsers/demo-JSON/JSON_grammar.rb
+- examples/parsers/demo-JSON/JSON_lexer.rb
+- examples/parsers/demo-JSON/JSON_parser.rb
 - examples/parsers/parsing_abc.rb
 - examples/parsers/parsing_ambig.rb
 - examples/parsers/parsing_b_expr.rb