grammaphone 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a6fc51b323812f7278f9e51540cccbc1df3530c17da6abc75f9fb7c97065433d
+  data.tar.gz: 3e17ab606b44b15d37445d81a73102fb2122569ecc43839c40c25498f6ebd340
+SHA512:
+  metadata.gz: b06c48105fd3b8b3682fe66fa1ec747d779100174724dd2b17de1c316e9b6dc81ff64aedebda0c9c97fa5654c7a7bf9767d4d07d9f15bd59f7d13e354d6f977f
+  data.tar.gz: 492dce702c1fa07ce1258c5ff62c32bfc8c8669ec224def19fba81eb8fc7ee69b83f7ca6d8b8518ab7b4d5e491538efbe32c224cea635f80f8d3008a0ad16aee

data/lib/grammaphone.rb

@@ -0,0 +1,192 @@
+# Grammaphone is a dynamically-definable parser pseudo-generator based on a 
+# BNF-like grammar.
+#
+# ### Grammar
+# A grammar is defined using key-value pairs in a hash. This is viewed as a 
+# names with associated sets of patterns it can match against. A name can be 
+# any sequence of characters that are valid in a Ruby String (i.e. any Unicode 
+# character). Similarly, the patterns can be composed of any Ruby-valid characters.
+#
+# ### Writing a Rule
+# A rule is a list of element identifiers, separated by spaces (`" "` or `\x20`).
+#
+# Element identifiers fall into three categories: literals, patterns, and rules.
+#
+# A literal is a string of characters that will be matched iff the token matches 
+# the literal text *exactly*. Literal sequences are preceeded by one double-quote 
+# (`"` or `\x22`). For example, to match the exact string "hello", the literal 
+# string you would use is `"hello`. Note that the initial double-quote is not 
+# included in the literal itself.
+#
+# A pattern is a string of characters representing a Ruby-valid regex. Pattern 
+# sequences are surrounded by one forward slash (`"/"` or `\x2F`) on each end. 
+# For example, to match a capitalized name composed strictly of ASCII letters, 
+# (e.g. "April", "John", "Alex"), the pattern string could be `/[A-Z][a-z]*/`. 
+# Note that every pattern must strictly match the whole token. Anything else 
+# won't be matched.
+#
+# A rule identifier is a string of characters representing the name of a grammatical 
+# rule. Rule identifier sequences are trivial, and are specified by using the 
+# exact name of the rule with no decorations. For example, to reference a rule 
+# named "NUMBER", the Rule string is `NUMBER`. Note that, like any grammar, a rule 
+# can refer to itself to define a recursive pattern. If a rule name is specified 
+# that doesn't exist in the current grammar, an exception will be raised and 
+# parsing will immediately stop.
+#
+# A rule is composed of zero or more element identifiers, which are evaluated in 
+# order. If a rule has no identifiers, then it will only match an empty token 
+# list, which will always succeed.
+#
+# Multiple options for a rule can be specified by passing an Array where each 
+# element of the Array is a valid rule. These rules are treated as possibilities 
+# for matching, with a precedence specified by the order.
+#
+# #### Example
+# The two most common introductory programs are "Hello, world!", and an 
+# introduction program, given a name. For the purposes of this example, the latter
+# prints in the format "Hello, \<name\>!", where `<name>` is the name entered.
+#
+# The following Hash describes the grammar that matches the output of these programs, 
+# assuming they are tokenized as ["Hello", ",", " ", \<name\>/world, "!"]. That 
+# tokenization is not default, but is assumed for the purposes of this example. 
+# This is by no means the only possible grammar, just an example.
+#
+# ```ruby
+# {
+#   START: '"Hello ", /\s/ NAME "!',
+#   NAME: ['"world', '/[A-Z][a-z]*/']
+# }
+# ```
+#
+# Note that to match a space, you need to use the pattern, since the splitting function 
+# for rules splits on the space character, regardless of where it is.
+
+require_relative "grammaphone/errors"
+require_relative "grammaphone/tokens"
+require_relative "grammaphone/rule"
+
+class Grammaphone
+
+  def self.tokenize(src, &split_method)
+    TokenStream.new(src, &split_method)
+  end
+
+  # node_type must accept a 
+  def initialize(rules = {}, node_type = Array, &default_action)
+    raise ArgumentError.new("cannot form parser from a #{rules.class}") unless rules.kind_of? Hash
+    raise ArgumentError.new("syntax tree type must respond to <<") unless node_type.method_defined?(:"<<")
+    @default_action = (default_action.nil? ? lambda{|node, name| node} : default_action)
+    @node_type = node_type
+    @rules = rules.map do |k, v|
+      Rule.new(k, v, @default_action)
+    end
+  end
+
+  def add_rule(name, rule, &action)
+    m = @rules.find {|r| r.name == name}
+    action = @default_action if action.nil?
+    if m.nil?
+      @rules << Rule.new(name, rule, action)
+    else
+      m.rule = rule
+      m.action = action
+    end
+  end
+
+  def rules
+    @rules.map{|r| [r.name, r.rule]}.to_h
+  end
+
+  def parse(token_stream)
+    token_stream = TokenStream.new(token_stream) unless token_stream.kind_of?(TokenStream)
+    raise EmptyRulesetError if @rules.size == 0
+    res = self.send(@rules[0].name, token_stream, @node_type)
+    res
+  end
+
+  # Not to be released in shipped version
+  def test(name, token_stream)
+    self.send(name, TokenStream.new(token_stream))
+  end
+
+  def respond_to_missing?(m, include_all)
+    (include_all && @rules.any?{|r| r.name == m}) || super
+  end
+
+  # This is fun, but it doesn't really take advantage of metaprogramming in a way 
+  # that can't be accomplished with match_rule. It also lets the rules be "called" 
+  # outside of normal context
+  def method_missing(m, *args, &block)
+    r = @rules.find{|r| r.name == m}
+    if r
+      match_rule(r, args[0], args[1])
+    else
+      super
+    end
+  end
+  
+  private
+ 
+  def match_rule(r, stream, result_type)
+    # This is an enormous function. It needs to be pared down
+    matches = nil
+    result = result_type.new
+    r.each do |option|
+      tokens = stream.dup
+      break if option.empty?
+      matched = true
+
+      option.each do |element|
+        token = tokens.peek
+        # puts "rule: #{r.name}; element: #{element}; token: #{token}"
+        if Token.literal?(element)
+          unless Token.matches_literal?(element, token)
+            matches = nil
+            matched = false
+            break
+          end
+
+          matches ||= []
+          matches << token
+          result << token
+          tokens.next # might as well be tokens.skip
+        elsif Token.pattern?(element)
+          unless Token.matches_pattern?(element, token)
+            matches = nil
+            matched = false
+            break
+          end
+
+          matches ||= []
+          unless token.nil?
+            matches << token
+            result << token
+          end
+          tokens.next
+        else
+          raise TokenError.new("Can't have empty patterns") if element.empty?
+
+          submatches, res = self.send(element, tokens, result_type)
+          unless submatches
+            matches = nil
+            matched = false
+            break
+          end
+
+          matches ||= []
+          matches << submatches
+          result << res
+          tokens.skip([submatches.size, 1].max)
+        end
+      end
+
+      if matched
+        result = r.trigger(result)
+        break
+      end
+    end
+    # puts "matches for rule #{r.name}: #{matches.to_s}" unless matches.nil?
+    return false if matches.nil?
+    [matches, result]
+  end
+end

data/lib/grammaphone/errors.rb

@@ -0,0 +1,29 @@
+class Grammaphone
+  class ParseError < StandardError; end
+
+  class RulesetError < ParseError
+    def message
+      super + "Problem with ruleset definition"
+    end
+  end
+
+  class EmptyRulesetError < RulesetError
+    def message
+      super + ": empty ruleset not allowed"
+    end
+  end
+
+  class TokenError < ParseError; end
+
+  class NonstringTokenError < TokenError
+    def message
+      super + "Token not a String"
+    end
+  end
+
+  class TokenStreamError < TokenError
+    def message
+      super + "Non-Array-able types can't be tokenized"
+    end
+  end
+end

data/lib/grammaphone/rule.rb

@@ -0,0 +1,60 @@
+require_relative "./errors"
+class Grammaphone
+  private
+  
+  class Rule
+    attr_reader :name
+
+    def initialize(name, rule, act = nil)
+      raise ArgumentError.new("rule names must be a String or Symbol") unless (name.kind_of?(Symbol) || name.kind_of?(String))
+      @name = name.to_sym
+      self.rule = rule
+      self.action = act
+    end
+
+    def rule
+      @rule.dup
+    end
+
+    def rule=(rule)
+      case rule
+      when Array
+        raise ArgumentError.new("grammar rule as an Array must contain only Strings") unless rule.all?{|r| r.kind_of?(String)}
+        @rule = rule.dup
+      when String
+        @rule = [rule.dup]
+      else
+        raise ArgumentError.new("grammar rule must be a String or Array of Strings")
+      end
+      @allows_empty = @rule.any?{|r| r.empty?}
+    end
+
+    # action expected to return an Array-like object with flatten implemented
+    def action=(action)
+      raise ArgumentError.new("rule actions must be a proc") unless (action.kind_of?(Proc) || action.kind_of?(NilClass))
+      if action.nil?
+        @action = lambda {|tokens, name| token}
+      else
+        @action = action
+      end
+    end
+
+    def each 
+      if block_given?
+        @rule.each do |r|
+          yield r.split(" ")
+        end
+      else
+        to_enum(:each)
+      end
+    end
+
+    def allows_empty?
+      @allows_empty
+    end
+
+    def trigger(node)
+      @action.call(node, name)
+    end
+  end
+end

data/lib/grammaphone/tokens.rb

@@ -0,0 +1,141 @@
+require_relative "./errors"
+class Grammaphone
+  # This is not a descendant of Enumerator. This is explicit and intentional, 
+  # due to use as an almost tree-like object. This implementation makes it behave 
+  # as a near-functional list structure, which is extremely useful for this parser.
+  class TokenStream
+    # This doesn't need to be here, but it could potentially be useful
+    include Enumerable
+
+    def initialize(tokens, &split_method)
+      case tokens
+      when String
+        if split_method.nil?
+          @enum = tokens.split(" ")
+        else
+          @enum = split_method.call(tokens).to_a
+        end
+      when Array
+        @enum = tokens.dup
+      else
+        raise TokenStreamError unless tokens.respond_to?(:to_a)
+        @enum = tokens.to_a.dup # dup just in case
+      end
+      @pointer = 0
+    end
+
+    # This ensures that all instances refer to the exact same token stream, 
+    # but not necessarily at the same point. This saves a great deal of 
+    # memory, without risking stream data integrity.
+    def initialize_copy(orig)
+      @enum = orig.instance_variable_get(:@enum)
+      super
+    end
+
+    # Gets the next non-empty token, consuming all viewed tokens.
+    #
+    # Follows the same relationship as `peek` and `peek_token`
+    def next
+      token = next_token
+      token = next_token while token&.empty?
+      token
+    end
+
+    # Gets the next token, consuming it.
+    def next_token
+      token = @enum[@pointer]
+      raise NonstringTokenError unless token.nil? || token.kind_of?(String) 
+      @pointer += 1
+      token
+    end
+
+    # Peeks at the nth token from the current pointer, not counting empty tokens, 
+    # not consuming any tokens.
+    #
+    # if no count is given, deaults to the next immediate token.
+    #
+    # Follows the same relationship as `next` and `next_token`
+    def peek(n = 0)
+      offset = (0..n).inject(0) do |acc, p|
+        peek_token(p)&.empty? ? acc + 1 : acc
+      end
+      peek_token(n + offset)
+    end
+
+    # Peeks at the nth token from the current pointer, not consuming it.
+    #
+    # If no count is given, defaults to the next immediate token.
+    def peek_token(n = 0)
+      raise ArgumentError.new("can't look back in the token stream") if n < 0
+      @enum[@pointer + n]
+    end
+
+    # Consumes the next n tokens, returning `self`.
+    #
+    # This has no meaningful effect if the stream is empty.
+    #
+    # If no count is given, defaults to consuming a single token
+    def skip(n = 1)
+      @pointer += n
+      self
+    end
+
+    # Resets the pointer to the beginning of the token stream.
+    def reset
+      @pointer = 0
+      self
+    end
+
+    # Returns `true` if there are no tokens remaining in the stream and `false` 
+    # otherwise. That is, any calls to `peek_token`, `peek`, `next_token`, or 
+    # `next` are guaranteed to return `nil` if `empty?` returns `true`.
+    def empty?
+      @pointer >= @enum.size
+    end
+
+    # Provided because there's a chance that it'll be useful. At the very least, 
+    # it can't hurt, since any arrays produced are copies.
+    def each
+      if block_given?
+        @enum.each { |t| yield t }
+        self
+      else
+        to_enum(:each)
+      end
+    end
+
+    # Returns the remaining tokens as an Array.
+    def to_a
+      @enum[@pointer..].dup
+    end
+  end
+
+  module Token
+    LITERAL_PREFIX = "\""
+
+    def self.literal?(token)
+      token[0] == LITERAL_PREFIX
+    end
+
+    def self.clean_literal(token)
+      token[1..]
+    end
+
+    def self.matches_literal?(element, token)
+      !token.nil? && literal?(element) && token == clean_literal(element)
+    end
+
+    def self.pattern?(token)
+      token[0] == "/" && token[-1] == "/"
+    end
+
+    def self.clean_pattern(token)
+      /\A#{token[1...-1]}\Z/
+    end
+
+    def self.matches_pattern?(element, token)
+      pattern?(element) && (token =~ clean_pattern(element)) ||
+        token.nil? && "" =~ clean_pattern(element)
+    end
+  end
+end

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: grammaphone
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kellen Watt
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-09-15 00:00:00.000000000 Z
+dependencies: []
+description: A dynamic parser written in Ruby that uses a BNF-adjascent grammar.
+email: kbw6d9@mst.edu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/grammaphone.rb
+- lib/grammaphone/errors.rb
+- lib/grammaphone/rule.rb
+- lib/grammaphone/tokens.rb
+homepage: https://github.com/KellenWatt/grammaphone
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: A pure Ruby dynamic parser
+test_files: []