yoga 0.0.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6a3b73d2db5477d58eab02a2179a42888bc0ba2
-  data.tar.gz: 49c440508979558a68d01fb44142b267eae7e176
+  metadata.gz: 4999074c914d1566ee18799548c1d437ff5ad83b
+  data.tar.gz: 998165e1b73a1d6fef8350b7a5b5f15bfb6295ee
 SHA512:
-  metadata.gz: 8a9cac22e56f5c19de82a070d83e94a3a825c5f4666c495d5e9dbe0d5bea87543534acbade0b545ed929985178f4508559a4cb5d3ae5f9278f5c23b3add27231
-  data.tar.gz: 3e374cd507d6e4d44429300ea4bea0d4f4752c79570ccc6d8117a5ba280193ee2ff1ef0acce064cb4af13247e6b719c141d851aa8ef7026d8dbf2adc5c8eab37
+  metadata.gz: 3f0f59113580eac2ac0065651837f3e0bef3044585e48ff61e92c8b9a3cc21faac33a45ab0a98ed908d80de05adcc5bafb66a1e291ca6150c0641c25fbe25947
+  data.tar.gz: 9c86dc7896453ebe2e6c5bdb060f664757f9246af44a4e3ff556789b5c0851fe7648f53b8c493d3254babea180195e356226c1d544933b1382d2875e359fbf26

data/.gitignore

@@ -6,8 +6,9 @@
 Gemfile.lock
 InstalledFiles
 _yardoc
-coverage
+coverage/
 doc/
+profile/
 lib/bundler/man
 pkg
 rdoc

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+--require pry

data/.rubocop.yml

@@ -0,0 +1,40 @@
+AllCops:
+  TargetRubyVersion: 2.3
+
+Metrics/LineLength:
+  Enabled: false
+Metrics/MethodLength:
+  Max: 15
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/*.rb'
+    - 'spec/**/*.rb'
+Metrics/AbcSize:
+  Max: 25
+
+Style/AlignParameters:
+  EnforcedStyle: with_fixed_indentation
+Style/SignalException:
+  EnforcedStyle: only_fail
+Style/AccessModifierIndentation:
+  EnforcedStyle: outdent
+Style/Documentation:
+  Enabled: false
+Style/Encoding:
+  Enabled: true
+  EnforcedStyle: always
+  AutoCorrectEncodingComment: "# encoding: utf-8\n"
+Style/RegexpLiteral:
+  EnforcedStyle: mixed
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+Style/AlignHash:
+  EnforcedLastArgumentHashStyle: ignore_implicit
+Style/AlignArray:
+  Enabled: false
+Style/IndentArray:
+  Enabled: false
+Style/Alias:
+  EnforcedStyle: prefer_alias_method
+Style/ParallelAssignment:
+  Enabled: false

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.4.0
+  - 2.2.2
+  - 2.1.0
+script:
+  - bundle exec rubocop --format clang
+  - bundle exec rspec spec

data/.yardopts

@@ -0,0 +1,3 @@
+-m markdown
+--protected
+--private

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at me@medcat.me. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -1,4 +1,11 @@
-source 'https://rubygems.org'
+# encoding: utf-8
+# frozen_string_literal: true
+
+source "https://rubygems.org"
 
 # Specify your gem's dependencies in yoga.gemspec
 gemspec
+
+gem "mixture"
+gem "pry"
+gem "pry-stack_explorer"

data/README.md

@@ -1,12 +1,16 @@
 # Yoga
 
-TODO: Write a gem description
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/yoga`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
 
 ## Installation
 
 Add this line to your application's Gemfile:
 
-    gem 'yoga'
+```ruby
+gem 'yoga'
+```
 
 And then execute:
 
@@ -20,10 +24,13 @@ Or install it yourself as:
 
 TODO: Write usage instructions here
 
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/yoga/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/yoga. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+

data/Rakefile

@@ -1,2 +1,9 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
 
+task default: :spec

data/lib/yoga.rb

@@ -1,5 +1,16 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
 require "yoga/version"
+require "yoga/errors"
+require "yoga/utils"
+require "yoga/location"
+require "yoga/node"
+require "yoga/parser"
+require "yoga/scanner"
+require "yoga/token"
 
+# A parser and scanner helper.
 module Yoga
   # Your code goes here...
 end

data/lib/yoga/errors.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  # An error originating from the Yoga class.  All behavior here is private
+  # to the Yoga module.
+  #
+  # @api private
+  # @private
+  class Error < ::StandardError
+    # Initialize the error.
+    #
+    # @api private
+    def initialize(data)
+      data.each { |k, v| instance_variable_set(:"@#{k}", v) }
+      super(generate_message)
+    end
+
+  private
+
+    # Generates a message for the exception.
+    #
+    # @return [::String]
+    def generate_message
+      message
+    end
+  end
+
+  # An error that has a corresponding location information.
+  #
+  # @api private
+  class LocationError < Error
+    attr_reader :location
+  end
+
+  # An error that occurred with parsing.
+  #
+  # @api private
+  class ParseError < LocationError; end
+
+  # An unexpected token was encountered while parsing.
+  #
+  # @api private
+  class UnexpectedTokenError < ParseError
+    # (see Error#generate_message)
+    private def generate_message
+      "Unexpected #{@got}, expected one of #{@expected.to_a.join(', ')} " \
+        "at #{@location}"
+    end
+  end
+
+  # A shift was requested, but was invalid.
+  #
+  # @api private
+  class InvalidShiftError < ParseError
+    # (see Error#generate_message)
+    private def generate_message
+      "Parser shifted, but no tokens remain at #{@location}"
+    end
+  end
+end

data/lib/yoga/location.rb

@@ -0,0 +1,178 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  # A location in a file.  This can be used for debugging purposes.
+  class Location
+    # The file the location is positioned in.  This should just be a string
+    # that uniquely identifies the file from all possible values of the
+    # location.
+    #
+    # @return [::String]
+    attr_reader :file
+
+    # The line the location is on.  This can be a range of lines, or a single
+    # line.
+    #
+    # @return [::Range]
+    attr_reader :line
+
+    # The column the location on.  This can be a range of columns, or a single
+    # column.
+    #
+    # @return [::Range]
+    attr_reader :column
+
+    # A "hash" of the location.  This is a number that is meant to roughly
+    # represent the value of this location.  Used primarily for the Hash
+    # class.
+    #
+    # @api private
+    # @return [::Numeric]
+    attr_reader :hash
+
+    # Creates a "default" location.  This is a location that can
+    # be given if the location is unknown.
+    #
+    # @param file [::String] The file.  See {#file}.
+    # @return [Location]
+    def self.default(file = "<unknown>")
+      new(file, 0..0, 0..0)
+    end
+
+    # Initialize the location with the given information.
+    #
+    # @param file [::String] The file.  See {#file}.
+    # @param line [::Range, ::Numeric] The line.  See {#line}.
+    # @param column [::Range, ::Numeric] The column.  See {#column}.
+    def initialize(file, line, column)
+      @file = file.freeze
+      @line = ensure_range(line).freeze
+      @column = ensure_range(column).freeze
+      @hash = [@file, @line, @column].hash
+      freeze
+    end
+
+    # Creates a string representing this location in a file.
+    #
+    # @return [::String]
+    def to_s
+      "#{file}:#{range(line)}.#{range(column)}"
+    end
+
+    # Pretty inspect.
+    #
+    # @return [::String]
+    def inspect
+      "#<#{self.class} #{self}>"
+    end
+
+    # Determines if the other object is equal to the current instance.  This
+    # checks `equal?` first to determine if they are strict equals; otherwise,
+    # it checks if the other is a {Location}.  If it is, it checks that the
+    # properties are equal.
+    #
+    # @example
+    #   a # => #<Location file="a" line=1..1 column=5..20>
+    #   a == a # => true
+    # @example
+    #   a # => #<Location file="a" line=1..1 column=5..20>
+    #   b # => #<Location file="a" line=1..1 column=5..20>
+    #   a == b # => true
+    # @example
+    #   a # => #<Location file="a" line=1..1 column=5..20>
+    #   b # => #<Location file="b" line=1..1 column=6..20>
+    #   a == b # => false
+    #
+    # @param other [Object]
+    # @return [Boolean]
+    def ==(other)
+      equal?(other) || other.is_a?(Location) && @file == other.file &&
+        @line == other.line && @column == other.column
+    end
+
+    # Unions this location with another location.  This creates a new location,
+    # with the two locations combined.  A conflict in the file name causes an
+    # error to be raised.
+    #
+    # @example
+    #   a = Location.new("a", 1..1, 5..10)
+    #   a.union(a) # => #<Location file="a" line=1..1 column=5..10>
+    #   a.union(a) == a # => true
+    #   a.union(a).equal?(a) # => false
+    # @example
+    #   a = Location.new("a", 1..1, 5..10)
+    #   b = Location.new("a", 1..1, 6..20)
+    #   a.union(b) # => #<Location file="a" line=1..1 column=5..20>
+    # @example
+    #   a = Location.new("a", 1..5, 3..10)
+    #   b = Location.new("b", 1..3, 2..20)
+    #   a.union(b) # => #<Location file="a" line=1..5 column=2..20>
+    # @example
+    #   a # => #<Location ...>
+    #   b # => #<Location ...>
+    #   a.union(b) == b.union(a)
+    #
+    # @raise [::ArgumentError] if other isn't a {Location}.
+    # @raise [::ArgumentError] if other's file isn't the receiver's file.
+    # @param others [Location]
+    # @return [Location]
+    def union(*others)
+      others.each do |other|
+        fail ArgumentError, "Expected #{self.class}, got #{other.class}" \
+          unless other.is_a?(Location)
+        fail ArgumentError, "Expected other to have the same file" unless
+          file == other.file
+      end
+
+      line = construct_range([@line, *others.map(&:line)])
+      column = construct_range([@column, *others.map(&:column)])
+
+      Location.new(@file, line, column)
+    end
+
+    alias_method :|, :union
+
+  private
+
+    # Creates a range from a list of ranges.  This takes the lowest starting
+    # value, and the greatest ending value, and creates a new range from that.
+    #
+    # @param from [<::Range<::Numeric>>]
+    # @return [::Range<::Numeric>]
+    def construct_range(from)
+      first = from.map(&:first).min
+      last = from.map(&:last).max
+
+      first..last
+    end
+
+    # Ensures that the given value is a range.  If it's numeric, it's turned
+    # into a range with the same starting and ending values.  If it's a range,
+    # it's returned.  Otherwise, it fails.
+    #
+    # @param value [::Numeric, ::Range] The value to turn into a range.
+    # @return [::Range]
+    def ensure_range(value)
+      case value
+      when ::Numeric
+        value..value
+      when ::Range
+        value
+      else
+        fail ArgumentError, "Unexpected #{value.class}, expected Range"
+      end
+    end
+
+    # Returns a string version of the range.  If there is no distance between
+    # the starting and ending for the given range, it returns a string of the
+    # value.  Otherwise, it returns `<starting>-<ending>`.
+    #
+    # @param value [::Range<::Numeric>]
+    # @return [::String]
+    def range(value)
+      return value.first.to_s unless value.first != value.last
+      "#{value.first}-#{value.last}"
+    end
+  end
+end

data/lib/yoga/node.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require "mixture"
+
+module Yoga
+  # A parser node.  This can be subclassed, or used as is.
+  class Node
+    include Mixture::Model
+
+    # @!attribute [r] location
+    #   The location of the node.  This is normally dependant on the locations
+    #   of the tokens that make up this node.
+    #
+    #   @return [Yoga::Location]
+    attribute :location, type: Yoga::Location
+
+    # Initialize the node with the given attributes.  The `:kind` and
+    # `:location` attributes are required.
+    #
+    # @param attributes [Hash] The attributes.
+    # @option attributes [::Symbol] :kind The kind of the node.
+    # @option attributes [Yoga::Location] :location The location of the node.
+    def initialize(attributes)
+      self.attributes = attributes
+      freeze
+    end
+
+    # Creates a new node with the updated attributes.  If any unknown
+    # attributes are used, it fails.
+    #
+    # @param (see #initialize)
+    # @option (see #initialize)
+    def update(attributes)
+      self.class.new(self.attributes.merge(attributes))
+    end
+
+    # Returns whether or not the other object is equal to this one.
+    # If the other object is this opbject, it returns true; otherwise,
+    # if the other object is an instance of this object's class, and
+    # the attributes are equal, it returns true; otherwise, it returns
+    # false.
+    #
+    # @param other [::Object]
+    # @return [Boolean]
+    def ==(other)
+      equal?(other) || (other.is_a?(self.class) && \
+        attributes == other.attributes)
+    end
+
+    # Prevents all calls to {#update}.  This is used on nodes that should
+    # never be updated.  This is a stop-gap measure for incorrectly
+    # configured projects.  This, in line with {#update}, creates
+    # a duplicate node.
+    #
+    # @return [Yoga::Node]
+    def prevent_update
+      node = dup
+      node.singleton_class.send(:undef_method, :update)
+      node
+    end
+  end
+end

data/lib/yoga/parser.rb

@@ -0,0 +1,33 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require "yoga/parser/helpers"
+
+module Yoga
+  # A parsing helper.
+  #
+  # This uses the `@tokens` and `@_root` instance variables.
+  module Parser
+    # Initialize the parser.
+    #
+    # @param tokens [::Enumerable<Yoga::Token>]
+    def initialize(tokens)
+      @tokens = tokens
+    end
+
+    # Performs the parsing.
+    #
+    # @return [Yoga::Node]
+    def call
+      @_root ||= parse_root
+    end
+
+    # Internal ruby construct.
+    #
+    # @private
+    # @api private
+    def self.included(base)
+      base.send :include, Parser::Helpers
+    end
+  end
+end

data/lib/yoga/parser/helpers.rb

@@ -0,0 +1,234 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  module Parser
+    # helpers that manage the state of the parser.  This can be things like
+    # peeking, shifting, and erroring.
+    module Helpers
+      # The class methods for helpers for the parser.  These are defined
+      # on the class of the parser.
+      module ClassMethods
+        # The default value for a switch.  This matches no tokens and
+        # performs no actions.  This cannot be modified.
+        #
+        # @return [{::Symbol => ::Proc}]
+        DEFAULT_SWITCH = {}.freeze
+
+        # Predefines a switch logic.  This is used to do set checking for the
+        # keys instead of array checking.  This reduces the time it takes for
+        # a switch statement to be performed.
+        #
+        # @param name [::Symbol] The name of the switch.  This is used in the
+        #   method to look up the switch.
+        # @param mapping [{::Symbol, <::Symbol> => ::Proc}, nil] A
+        #   mapping of symbol(s) to callables.  If no value is given, then it
+        #   returns the current switch for the given name.
+        # @return [{::Symbol => {::Symbol => ::Proc}}]
+        #
+        # @overload switch(name)
+        #   Retrieves the switch map for the given name.  This is basically
+        #   a lookup table for the peek token.  The keys represent the peek
+        #   token kind, and the values are the actions that should be taken
+        #   for the given token kind.
+        #
+        #   @param name [::Symbol] The name of the switch map.  This is
+        #     typically the same as the first set name and the node name for
+        #     the rule.
+        #   @return [{::Symbol => ::Proc}] The switch map.
+        # @overload switch(name, mapping)
+        #   Sets the switch map for the given name.  This does some internal
+        #   mapping before finalizing the mapping.  First, it converts all
+        #   values into a proc; then, it expands all array (or enumerable)
+        #   keys into seperate entries.  It then sets the first set using the
+        #   name of the switch as the name of the first set, and the keys of
+        #   the switch map as the first set.
+        #
+        # @example
+        #   Parser.switch(:Definition,
+        #     :":" => proc { parse_directive },
+        #     :enum => proc { parse_enum },
+        #     :def => proc { parse_function },
+        #     :class => proc { parse_class },
+        #     :struct => proc { parse_struct })
+        #
+        #   @param name [::Symbol] The name of the switch map.  A first set
+        #     with the same name is created as well.
+        #   @param mapping [{::Symbol, ::Enumerable<::Symbol> =>
+        #     ::Proc, ::Symbol}] The mapping.
+        #   @return [void]
+        def switch(name, mapping = nil)
+          @_switches ||= {}
+          name = name.intern
+          return @_switches[name] || DEFAULT_SWITCH unless mapping
+          switch = {}
+          mapping.each do |k, v|
+            v = v.is_a?(::Proc) ? v : proc { |*a| send(v, *a) }
+            Array(k).each { |n| switch[n] = v }
+          end
+
+          first(name, switch.keys)
+          @_switches[name] = switch
+        end
+
+        # The first sets.  The key is the name of a node, and the value
+        # is a set of all tokens that can be at the start of that node.
+        #
+        # @return [{::Symbol => ::Set<::Symbol>}]
+        def firsts
+          @firsts ||= Hash.new { |h, k| h[k] = Set.new }
+        end
+
+        # @overload first(name)
+        #   Retrieves the first set for the given node name.  If none exists,
+        #   a `KeyError` is raised.
+        #
+        #   @api private
+        #   @param name [::Symbol] The name of the node to look up the first
+        #     set for.
+        #   @return [::Set<::Symbol>] The first set
+        #   @raise [::KeyError] If the node has no defined first set.
+        # @overload first(name, tokens)
+        #   Merges the given tokens into the first set of the name of the
+        #   node given.
+        #
+        #   @api private
+        #   @param name [::Symbol] The name of the node that the first set is
+        #     for.
+        #   @param tokens [<::Symbol>] The tokens that act as the first set
+        #     for the node.
+        #   @return [void]
+        def first(name, tokens = nil)
+          if tokens
+            firsts[name] = Set.new(Array(tokens))
+          else
+            firsts.fetch(name)
+          end
+        end
+      end
+
+      # Peeks to the next token.  If peeking would cause a `StopIteration`
+      # error, it instead returns the last value that was peeked.
+      #
+      # @return [Yoga::Token]
+      def peek
+        if next?
+          @_last = @tokens.peek
+        else
+          @_last
+        end
+      end
+
+      # Checks if the next token exists.  It does this by checking for a
+      # `StopIteration` error.  This is actually really slow, but there's
+      # not much else I can do.
+      #
+      # @return [::Boolean]
+      def next?
+        @tokens.peek
+        true
+      rescue StopIteration
+        false
+      end
+
+      # Switches the function executed based on the given nodes.  If the
+      # peeked node matches one of the mappings, it calls the resulting
+      # block or method.
+      #
+      # If the peeked token is not in the map, then {#error} is called.
+      #
+      # @see ClassMethods#switch
+      # @param name [::Symbol] The name of the switch to use.
+      # @return [::Object] The result of calling the block.
+      def switch(name, *param)
+        switch = self.class.switch(name)
+        block = switch.fetch(peek.kind) { error(switch.keys) }
+        instance_exec(*param, &block)
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+
+      # "Collects" a set of nodes until a terminating token.  It yields
+      # until the peek is the token.
+      #
+      # @param ending [::Symbol] The terminating token.
+      # @param join [::Symbol, nil] The token that joins each of the
+      #   children.  This is the comma between arguments.
+      # @return [::Array] The collected nodes from the yielding process.
+      def collect(ending, join = nil)
+        children = []
+        join = Utils.flatten_into_set([join]) if join
+        ending = Utils.flatten_into_set([ending]) if ending
+
+        return [] if (ending && peek?(ending)) || (!ending && !join)
+
+        children << yield
+        while (join && expect(join)) && !(ending && peek?(ending))
+          children << yield
+        end
+
+        children
+      end
+
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      # Checks to see if any of the given kinds includes the next token.
+      #
+      # @param tokens [<::Symbol>] The possible kinds.
+      # @return [::Boolean]
+      def peek?(*tokens)
+        tokens = Utils.flatten_into_set(tokens)
+        tokens.include?(peek.kind)
+      end
+
+      # Shifts to the next token, and returns the old token.
+      #
+      # @return [Yoga::Token]
+      def shift
+        @tokens.next
+      rescue ::StopIteration
+        fail InvalidShiftError, location: peek.location
+      end
+
+      # Sets up an expectation for a given token.  If the next token is
+      # an expected token, it shifts, returning the token; otherwise,
+      # it {#error}s with the token information.
+      #
+      # @param tokens [<::Symbol>] The expected tokens.
+      # @return [Yoga::Token]
+      def expect(*tokens)
+        tokens = Utils.flatten_into_set(tokens)
+        return shift if peek?(*tokens)
+        error(tokens.flatten)
+      end
+
+      # Retrieves the first set for the given node name.
+      #
+      # @param name [::Symbol]
+      # @return [::Set<::Symbol>]
+      def first(name)
+        self.class.first(name)
+      end
+
+      # Errors, noting the expected tokens, the given token, the location
+      # of given tokens.  It does this by failing.
+      #
+      # @param tokens [<::Symbol>] The expected tokens.
+      # @return [void]
+      def error(tokens)
+        fail Yoga::UnexpectedTokenError, expected: tokens, got: peek.kind,
+          location: peek.location
+      end
+
+      # Internal ruby construct.
+      #
+      # @private
+      # @api private
+      def self.included(base)
+        base.extend ClassMethods
+      end
+    end
+  end
+end

data/lib/yoga/scanner.rb

@@ -0,0 +1,156 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  # A scanner.  This performs scanning over a series of tokens.
+  # It is built to lazily scan whenever it is required, instead
+  # of all at once.  This integrates nicely with the parser.
+  module Scanner
+    # Initializes the scanner with the given source.  Once the
+    # source is set, it shouldn't be changed.
+    #
+    # @param source [::String] The source.
+    def initialize(source)
+      @source = source
+      @line = 1
+      @last_line_at = 0
+    end
+
+    # @overload call(&block)
+    #   For every token that is scanned, the block is yielded to.
+    #
+    #   @yieldparam token [Scanner::Token]
+    #   @return [self]
+    # @overload call
+    #   Returns an enumerable over the tokens in the scanner.
+    #
+    #   @return [::Enumerable<Scanner::Token>]
+    def call
+      return to_enum(:call) unless block_given?
+      @scanner = StringScanner.new(@source)
+      @line = 1
+
+      until @scanner.eos?
+        value = scan
+        yield value if value.is_a?(Token)
+      end
+
+      yield Token.eof(location)
+      self
+    end
+
+    # The scanning method.  This should return one of two values: a {Token},
+    # or `true`.  `nil` should _never_ be returned.  This performs an
+    # incremental scan of the document; it returns one token at a time.  If
+    # something matched, but should not emit a token, `true` should be
+    # returned.  The implementing class should mark this as private or
+    # protected.
+    #
+    # @abstract
+    #   Please implement this method in order to make the class a scanner.
+    # @return [Yoga::Token, true]
+    def scan
+      fail NotImplementedError, "Please implement #{self.class}#scan"
+    end
+
+  private
+
+    # Returns a location at the given location.  If a size is given, it reduces
+    # the column number by the size and returns the size from that.
+    #
+    # @example
+    #   @scanner.string # => "hello"
+    #   @line # => 1
+    #   @scanner.charpos # => 5
+    #   location # => #<Yoga::Location <anon>:1.6>
+    #   location(5) # => #<Yoga::Location <anon>:1.1-6
+    # @param size [::Numeric] The size of the token.
+    # @return [Yoga::Location]
+    def location(size = 0)
+      start = (@scanner.charpos - @last_line_at) + 1
+      column = (start - size)..start
+      Location.new(file, current_line, column)
+    end
+
+    # Creates a scanner token with the given name and source.  This grabs the
+    # location using {#location}, setting the size to the size of the source
+    # text.  The source is frozen before initializing the token.
+    #
+    # @example
+    #   emit(:<, "<") # => #<Yoga::Token kind=:< source="<">
+    # @return [Yoga::Token]
+    def emit(kind, source = @scanner[0])
+      Token.new(kind.freeze, source.freeze, location(source.length))
+    end
+
+    # Attempts to match the given token.  The first argument can be a string,
+    # a symbol, or a regular expression.  If the matcher is a symbol, it's
+    # coerced into a regular expression, with a forward negative assertion for
+    # any alphanumeric characters, to prevent partial matches (see
+    # {#symbol_negative_assertion}).  If the matcher is a regular expression,
+    # it is left alone.  Otherwise, `#to_s` is called and passed to
+    # `Regexp.escape`.  If the text is matched at the current position, a token
+    # is returned; otherwise, nil is returned.
+    #
+    # @param matcher [::Symbol, ::Regexp, #to_s]
+    # @param kind [::Symbol] The kind of token to emit.  This defaults to a
+    #   symbol version of the matcher.
+    # @return [Yoga::Token, nil]
+    def match(matcher, kind = :"#{matcher}")
+      matcher = case matcher
+                when ::Symbol then /#{::Regexp.escape(matcher.to_s)}#{symbol_negative_assertion}/
+                when ::Regexp then matcher
+                else /#{::Regexp.escape(matcher.to_s)}/
+                end
+
+      ((kind && emit(kind)) || true) if @scanner.scan(matcher)
+    end
+
+    # A regular expression to match all kinds of lines.  All of them.
+    #
+    # @return [::Regexp]
+    LINE_MATCHER = /\r\n|\n\r|\n|\r/
+
+    # Matches a line.  This is separate in order to allow internal logic,
+    # such as line counting and caching, to be performed.
+    #
+    # @return [Boolean] If the line was matched.
+    def match_line(kind = false)
+      match(LINE_MATCHER, kind).tap do |t|
+        break unless t
+        @line += 1
+        @last_line_at = @scanner.charpos
+      end
+    end
+
+    # Returns the number of lines that have been covered so far in the scanner.
+    # I recommend replacing this with an instance variable that caches the
+    # result of it, so that whenever you scan a new line, it just increments
+    # the line count.
+    #
+    # @return [::Numeric]
+    def current_line
+      # @scanner.string[0..@scanner.charpos].scan(/\A|\r\n|\n\r|\n|\r/).size
+      @line
+    end
+
+    # The negative assertion used for converting a symbol matcher to a regular
+    # expression.  This is used to prevent premature matching of other
+    # identifiers.  For example, if `module` is a keyword, and `moduleA` is
+    # an identifier, this negative assertion allows the following expression
+    # to properly match as such: `match(:module) || module(/[a-zA-Z], :IDENT)`.
+    #
+    # @return [#to_s]
+    def symbol_negative_assertion
+      "(?![a-zA-Z])"
+    end
+
+    # The file of the scanner.  This can be overwritten to provide a descriptor
+    # for the file.
+    #
+    # @return [::String]
+    def file
+      @file ||= "<anon>"
+    end
+  end
+end

data/lib/yoga/token.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  # A token that is used in scanning.  This is emitted when a lexical
+  # structure is encountered; e.g., for a `{`.
+  class Token
+    # The kind of the token.  For any explicit matches (e.g. `"{"`),
+    # this is the exact match (e.g. `:"{"`).  For any regular matches
+    # (e.g. an identifier), this is in all caps (e.g. `:IDENTIFIER`).
+    #
+    # @return [::Symbol]
+    attr_reader :kind
+
+    # The lexeme that this token is associated.  This is what the token
+    # matched directly from the source.
+    #
+    # @return [::String]
+    attr_reader :value
+
+    # The exact location the token was taken from.  This only spans one
+    # line.
+    #
+    # @return [Location]
+    attr_reader :location
+
+    # A "hash" of the token.  This is a number that is meant to roughly
+    # represent the value of this token.  Used primarily for the Hash
+    # class.
+    #
+    # @api private
+    # @return [::Numeric]
+    attr_reader :hash
+
+    # Creates an `EOF`-kind token.  This creates the token at the given
+    # location.
+    #
+    # @param location [Location]
+    # @return [Token]
+    def self.eof(location = Location.default)
+      new(:EOF, "", location)
+    end
+
+    # Initializes the token with the given kind, value, and location.
+    #
+    # @param kind [::Symbol] The kind of token.  See {#kind}.
+    # @param value [::String] The value of the token.  See {#value}.
+    # @param location [Location] The location of the token.  See {#location}.
+    def initialize(kind, value, location)
+      @kind     = kind
+      @value    = value.freeze
+      @location = location
+      @hash     = [@kind, @value, @location].hash
+      freeze
+    end
+
+    # Determines if this object is equal to another object.  It first checks
+    # for equality using `equal?`, then checks if the other is a token.  If
+    # the other object is a token, it checks that the attributes equate.
+    #
+    # @param other [::Object]
+    # @return [Boolean]
+    def ==(other)
+      equal?(other) || other.is_a?(Token) && @kind == other.kind &&
+        @value == other.value && @location == other.location
+    end
+  end
+end

data/lib/yoga/utils.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+module Yoga
+  # Utilities used with the Yoga module.
+  #
+  # @api private
+  module Utils
+  module_function
+
+    # Takes an array of tokens or set/array of tokens and turns it into a
+    # single set.
+    #
+    # @param tokens [<Yoga::Token, ::Set<Yoga::Token>, ::Enumerable>]
+    #   The array to flatten into a set.
+    # @return [::Set<Yoga::Token>]
+    def flatten_into_set(tokens)
+      flatten_into_array(tokens).to_set
+    end
+
+    # Takes an array of tokens or a set/array of tokens and turns it into a
+    # single array.
+    #
+    # @param tokens [<Yoga::Token>, ::Set<Yoga::Token>, ::Enumerable]
+    #   The array to flatten into a set.
+    # @return [<Yoga::Token>]
+    def flatten_into_array(tokens)
+      tokens.flat_map do |part|
+        if part.is_a?(::Enumerable)
+          flatten_into_array(part)
+        else
+          part
+        end
+      end
+    end
+  end
+end

data/lib/yoga/version.rb

@@ -1,3 +1,9 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
 module Yoga
-  VERSION = "0.0.1"
+  # The version of the module.
+  #
+  # @return [::String]
+  VERSION = "0.2.0"
 end

data/script/bench

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# frozen_string_literal: true
+
+require "benchmark/ips"
+require "ruby-prof"
+require "yoga/utils"
+
+VALUES = [
+  [[[[:x], :a], :b]],
+  [[Set[[:a], :x], :a]],
+  [[[[[[[[[[:x], :a], :c], :d], :e], :f], :d], :a], :a]]
+].freeze
+
+if ARGV[0] == "mark"
+  Benchmark.ips do |bench|
+    VALUES.each_with_index do |e, i|
+      bench.report(i) do |times|
+        i = 0
+        while i < times
+          Yoga::Utils.flatten_into_set(e)
+          i += 1
+        end
+      end
+    end
+  end
+else
+  result = RubyProf.profile do
+    VALUES.each do |e|
+      100_000.times do
+        Yoga::Utils.flatten_into_set(e)
+      end
+    end
+  end
+
+  printer = RubyProf::MultiPrinter.new(result)
+  printer.print(path: "profile/flatten", profile: "flatten")
+end

data/yoga.gemspec

@@ -1,23 +1,30 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# encoding: utf-8
+# frozen_string_literal: true
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'yoga/version'
+require "yoga/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "yoga"
   spec.version       = Yoga::VERSION
   spec.authors       = ["Jeremy Rodi"]
-  spec.email         = ["redjazz96@gmail.com"]
-  spec.summary       = %q{A lexer.}
-  spec.description   = %q{A lexer.}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.email         = ["me@medcat.me"]
 
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.summary       = "Ruby scanner and parser helpers."
+  spec.homepage      = "https://github.com/medcat/yoga"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.6"
-  spec.add_development_dependency "rake"
+  spec.add_dependency "mixture", "~> 0.6"
+
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "simplecov", "~> 0.13"
+  spec.add_development_dependency "rubocop", "~> 0.47"
 end

metadata

@@ -1,61 +1,130 @@
 --- !ruby/object:Gem::Specification
 name: yoga
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Jeremy Rodi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-16 00:00:00.000000000 Z
+date: 2017-03-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: mixture
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.47'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: A lexer.
+        version: '0.47'
+description: 
 email:
-- redjazz96@gmail.com
+- me@medcat.me
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- ".yardopts"
+- CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - lib/yoga.rb
+- lib/yoga/errors.rb
+- lib/yoga/location.rb
+- lib/yoga/node.rb
+- lib/yoga/parser.rb
+- lib/yoga/parser/helpers.rb
+- lib/yoga/scanner.rb
+- lib/yoga/token.rb
+- lib/yoga/utils.rb
 - lib/yoga/version.rb
+- script/bench
 - yoga.gemspec
-homepage: ''
-licenses:
-- MIT
+homepage: https://github.com/medcat/yoga
+licenses: []
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -73,9 +142,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
-summary: A lexer.
+summary: Ruby scanner and parser helpers.
 test_files: []
-has_rdoc: