rley 0.3.10 → 0.3.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20ceec50ffa7359dbf635a00184c604654cdf9bb
-  data.tar.gz: 76e9432f62d88127257c66277e21be37fd1b00bd
+  metadata.gz: ce491e5c66dbf214e515d360d74b17215783ba1b
+  data.tar.gz: 6527418c9d5070bf69b0d0f90d9463fb3d05354b
 SHA512:
-  metadata.gz: a80c8e38ede7dd3908a52b73fd8fa13fc7087537d8272c8d648764902cb29f0d5db23c157a2b704eb66e2c8322d7e2ee4822dfd36fadb25ac22e6ec9f517f3eb
-  data.tar.gz: 9d23a806c510790c6fcc19566520e11c9292d642b2debe20784c4a884d646c173a789ae3060345ad659698554fce6748078ef7d5cc2353f7e1c3495ecd156104
+  metadata.gz: d057009d0dccc8b7365021c6e1e1f93c1c34fac3b0b1bee16a8a0502950b5436715d0f1e7ded9f79ebcb484a43e3b1d2bd7b22b2dfe0dc64a52d47bc18745cd3
+  data.tar.gz: 92d1b41cb0ac96f65626b198920d60ab76bab3cf4925e74466083e286d9915503143424a9e63cf1593cd089ffd69ac5a25fe7ae95b8ddbbb2f9934c005a106bb

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+### 0.3.11 / 2016-12-04
+* [NEW] Directory `examples\data_formats\JSON`. A JSON demo parser.
+
 ### 0.3.10 / 2016-12-04
 * [NEW] Method `ParseForest#ambiguous?`. Indicates whether the parse is ambiguous.
 * [CHANGE] File `README.md` updated with new grammar builder syntax & typo fixes.

data/examples/data_formats/JSON/JSON_grammar.rb

@@ -0,0 +1,31 @@
+# Grammar for JSON data representation
+require 'rley'  # Load the gem
+
+
+########################################
+# Define a grammar for JSON
+builder = Rley::Syntax::GrammarBuilder.new do
+  add_terminals('KEYWORD') # For true, false, null keywords
+  add_terminals('JSON_STRING', 'JSON_NUMBER')
+  add_terminals('LACCOL', 'RACCOL') # For '{', '}' delimiters
+  add_terminals('LBRACKET', 'RBRACKET') # For '[', ']' delimiters
+  add_terminals('COLON', 'COMMA') # For ':', ',' separators
+  rule 'json_text' => 'json_value'
+  rule 'json_value' => 'json_object'
+  rule 'json_value' => 'json_array'
+  rule 'json_value' => 'JSON_STRING'
+  rule 'json_value' => 'JSON_NUMBER'
+  rule 'json_value' => 'KEYWORD'
+  rule 'json_object' => %w(LACCOL json_pairs RACCOL)
+  rule 'json_object' => %w(LACCOL RACCOL)
+  rule 'json_pairs' => %w(json_pairs COMMA single_pair)
+  rule 'json_pairs' => 'single_pair'
+  rule 'single_pair' => %w(JSON_STRING COLON json_value)
+  rule 'json_array' => %w(LBRACKET array_items RBRACKET)
+  rule 'json_array' => %w(LBRACKET RBRACKET)
+  rule 'array_items' => %w(array_items COMMA json_value)
+  rule 'array_items' => %w(json_value)
+end
+
+# And now build the grammar...
+GrammarJSON = builder.grammar

data/examples/data_formats/JSON/JSON_lexer.rb

@@ -0,0 +1,114 @@
+# File: JSON_lexer.rb
+# Lexer for the JSON data format
+require 'rley'  # Load the gem
+require 'strscan'
+
+# Lexer for JSON.
+class JSONLexer
+  attr_reader(:scanner)
+  attr_reader(:lineno)
+  attr_reader(:line_start)
+  attr_reader(:name2symbol)
+
+  @@lexeme2name = {
+    '{' => 'LACCOL',
+    '}' => 'RACCOL',
+    '[' => 'LBRACKET',
+    ']' => 'RBRACKET',
+    ',' => 'COMMA',
+    ':' => 'COLON'
+  }
+
+  class ScanError < StandardError ; end
+
+public
+  def initialize(source, aGrammar)
+    @scanner = StringScanner.new(source)
+    @name2symbol = aGrammar.name2symbol
+    @lineno =  1
+  end
+
+  def tokens()
+    tok_sequence = []
+    until @scanner.eos? do
+      token = _next_token
+      tok_sequence << token unless token.nil?
+    end
+
+    return tok_sequence
+  end
+
+private
+  def _next_token()
+    token = nil
+    skip_whitespaces
+    curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
+
+    begin
+      break if curr_ch.nil?
+
+      case curr_ch
+        when '{', '}', '[', ']', ',', ':'
+          type_name = @@lexeme2name[curr_ch]
+          token_type = name2symbol[type_name]
+          token = Rley::Parser::Token.new(curr_ch, token_type)
+
+        # LITERALS
+        when '"'  # Start string delimiter found
+          value = scanner.scan(/([^"\\]|\\.)*/)
+          end_delimiter = scanner.getch()
+          raise ScanError.new('No closing quotes (") found') if end_delimiter.nil?
+          token_type = name2symbol['JSON_STRING']
+          token = Rley::Parser::Token.new(value, token_type)
+
+        when /[ftn]/  # First letter of keywords
+          @scanner.pos = scanner.pos - 1 # Simulate putback
+          keyw = scanner.scan(/false|true|null/)
+          if keyw.nil?
+            invalid_keyw = scanner.scan(/\w+/)
+            raise ScanError.new("Invalid keyword: #{invalid_keyw}")
+          else
+            token_type = name2symbol['KEYWORD']
+            token = Rley::Parser::Token.new(keyw, token_type)
+          end
+
+
+        when /[-0-9]/ # Start character of number literal found
+          @scanner.pos = scanner.pos - 1 # Simulate putback
+          value = scanner.scan(/-?[0-9]+(\.[0-9]+)?([eE][-+]?[0-9])?/)
+          token_type = name2symbol['JSON_NUMBER']
+          token = Rley::Parser::Token.new(value, token_type)
+
+
+        else # Unknown token
+          erroneous = curr_ch.nil? ? '' : curr_ch
+          sequel = scanner.scan(/.{1,20}/)
+          erroneous += sequel unless sequel.nil?
+          raise ScanError.new("Unknown token #{erroneous}")
+      end #case
+
+
+    end while (token.nil? && curr_ch = scanner.getch())
+
+    return token
+  end
+
+
+  def skip_whitespaces()
+    matched = scanner.scan(/[ \t\f\n\r]+/)
+    return if matched.nil?
+
+    newline_count = 0
+    matched.scan(/\n\r?|\r/) { |_| newline_count += 1 }
+    newline_detected(newline_count)
+  end
+
+
+  def newline_detected(count)
+    @lineno += count
+    @line_start = scanner.pos()
+  end
+
+end # class
+
+

data/examples/data_formats/JSON/JSON_parser.rb

@@ -0,0 +1,47 @@
+# Purpose: to demonstrate how to build and render a parse tree for JSON
+# language
+require 'pp'
+require 'rley'  # Load the gem
+require_relative 'json_lexer'
+
+# Steps to render a parse tree (of a valid parsed input):
+# 1. Define a grammar
+# 2. Create a parser for that grammar
+# 3. Tokenize the input
+# 4. Let the parser process the input
+# 5. Generate a parse tree from the parse result
+# 6. Render the parse tree (in JSON)
+
+########################################
+# Step 1. Load a grammar for JSON
+require_relative 'JSON_grammar'
+
+# A JSON parser derived from our general Earley parser.
+class JSONParser < Rley::Parser::GFGEarleyParser
+  attr_reader(:source_file)
+
+  # Constructor
+  def initialize()
+    # Builder the Earley parser with the JSON grammar
+    super(GrammarJSON)
+  end
+  
+  def parse_file(aFilename)
+    tokens = tokenize_file(aFilename)
+    result = parse(tokens)
+
+    return result
+  end
+  
+  private
+  
+  def tokenize_file(aFilename)
+    input_source = nil
+    File.open(aFilename, 'r') { |f| input_source = f.read }
+
+    lexer = JSONLexer.new(input_source, GrammarJSON)
+    return lexer.tokens
+  end
+end # class
+
+# End of file

data/examples/data_formats/JSON/demo_json.rb

@@ -0,0 +1,31 @@
+require_relative 'JSON_parser'
+
+# Create a JSON parser object
+parser = JSONParser.new 
+
+# Parse the input file with name given in command-line
+if ARGV.empty?
+  msg = <<-END_MSG
+Command-line symtax:
+  ruby #{__FILE__} filename
+  where:
+    filename is the name of a JSON file
+    
+  Example:
+  ruby #{__FILE__} sample01.json
+END_MSG
+  puts msg
+  exit(1)
+end
+file_name = ARGV[0]
+result = parser.parse_file(file_name) # result object contains parse details
+
+unless result.success?
+  # Stop if the parse failed...
+  puts "Parsing of '#{file_name}' failed"
+  exit(1)
+end
+
+# Generate a parse forest from the parse result
+pforest = result.parse_forest
+# End of file

data/lib/rley/constants.rb

@@ -3,7 +3,7 @@
 
 module Rley # Module used as a namespace
   # The version number of the gem.
-  Version = '0.3.10'.freeze
+  Version = '0.3.11'.freeze
 
   # Brief description of the gem.
   Description = "Ruby implementation of the Earley's parsing algorithm".freeze

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rley
 version: !ruby/object:Gem::Version
-  version: 0.3.10
+  version: 0.3.11
 platform: ruby
 authors:
 - Dimitri Geshef
@@ -130,6 +130,10 @@ files:
 - README.md
 - Rakefile
 - examples/NLP/mini_en_demo.rb
+- examples/data_formats/JSON/JSON_grammar.rb
+- examples/data_formats/JSON/JSON_lexer.rb
+- examples/data_formats/JSON/JSON_parser.rb
+- examples/data_formats/JSON/demo_json.rb
 - lib/rley.rb
 - lib/rley/constants.rb
 - lib/rley/formatter/base_formatter.rb