c_lexer 2.5.1.0.pre1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3b6fdc822346ef189eb54aee2418b66806d4a1716483326a7f33be3953ef18b6
+  data.tar.gz: 74921a8413c03e7c5b388f8f7b1ea54e1e7d7579c613a608fe2b00f9f4355107
+SHA512:
+  metadata.gz: c4f17e98515d2f684b1dbde7427f96c7a52020a562b0d4c5d630066f1e0dc7c76bac279cdd8efc3d20da9b9593e0ec75aae8c40e7ae3f31f54dbb4177e6d3234
+  data.tar.gz: 8629e89c200a663e916b127338b8aeac648fc61b883e03be7a5f80b7622bdd5fb60429365cff579d837879d5944c2816d91d5a7a589b07a318b69b3a66716fb0

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+ext/lexer/lexer.c
+lib/lexer.bundle
+lib/lexer.so
+
+test.rb

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "parser"]
+	path = vendor/parser
+	url = git://github.com/whitequark/parser.git

data/.travis.yml

@@ -0,0 +1,7 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+  - 2.4.4
+  - 2.3.7
+before_install: gem install bundler

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| 'https://github.com/#{repo_name}' }
+
+# Specify your gem's dependencies in c_lexer.gemspec
+gemspec
+
+gem 'pry'
+gem 'parser', path: 'vendor/parser'

data/Gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: .
+  specs:
+    c_lexer (2.5.1.0.pre1)
+      ast (~> 2.4.0)
+      parser (= 2.5.1.0)
+
+PATH
+  remote: vendor/parser
+  specs:
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    cliver (0.3.2)
+    coderay (1.1.2)
+    docile (1.1.5)
+    json (2.1.0)
+    method_source (0.9.0)
+    minitest (5.11.3)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    racc (1.4.14)
+    rake (10.5.0)
+    rake-compiler (0.9.9)
+      rake
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.16)
+  c_lexer!
+  cliver (~> 0.3.2)
+  minitest (~> 5.10)
+  parser!
+  pry
+  racc (= 1.4.14)
+  rake (~> 10.0)
+  rake-compiler (~> 0.9)
+  simplecov (~> 0.15.1)
+
+BUNDLED WITH
+   1.16.1

data/README.md

@@ -0,0 +1,26 @@
+# CLexer
+
+[![Build Status](https://travis-ci.org/opal/c_lexer.svg?branch=master)](https://travis-ci.org/opal/c_lexer)
+
+A C port of the whitequark/parser's lexer.
+
+Initially based on [whitequark/parser#248](https://github.com/whitequark/parser/pull/248) (thanks [@alexdowad](https://github.com/alexdowad)).
+
+This project is still under development.
+
+## Installation
+
+```
+$ git submodule update --init
+$ bundle install
+$ rake
+```
+
+## Usage
+
+Simply require `c_lexer` after `parser` in your code.
+It should print `Using C lexer`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/opal/c_lexer.

data/Rakefile

@@ -0,0 +1,51 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rake/extensiontask'
+
+Rake::TestTask.new do |t|
+  t.ruby_opts  = ["-rpatch_helper"]
+  t.libs       = %w(lib/ test/ vendor/parser/test/ vendor/parser/lib/)
+  t.test_files = %w(vendor/parser/test/test_lexer.rb vendor/parser/test/test_parser.rb)
+  t.warning    = false
+end
+
+Rake::ExtensionTask.new('lexer')
+
+namespace :ruby_parser do
+  desc "'rake generate' in the Ruby Parser"
+  task :generate do
+    sh 'cd vendor/parser && rake generate'
+  end
+
+  desc "'rake clean' in the Ruby Parser"
+  task :clean do
+    sh 'cd vendor/parser && rake clean'
+  end
+end
+
+namespace :c_lexer do
+  desc 'Generate lexer.c from lexer.rl'
+  task :generate do
+    source = 'ext/lexer/lexer.rl'
+    target = 'ext/lexer/lexer.c'
+
+    sh "ragel -F1 #{source} -o #{target}"
+
+    # Ragel likes to use int variables where a #define would do
+    src = File.read(target)
+    src.gsub!(/^static const int (\w+) = (\d+);/, '#define \1 \2')
+    File.open(target, 'w') { |f| f.write(src) }
+  end
+end
+
+task :clean do
+  sh 'rm -rf tmp'
+  sh 'rm -f lib/lexer.*'
+  sh 'rm -f ext/lexer/lexer.c'
+  sh 'cd vendor/parser && rake clean'
+end
+
+task generate: ['ruby_parser:generate', 'c_lexer:generate']
+task test: [:generate, :compile]
+task build: :generate
+task default: :test

data/bin/c-ruby-parse

@@ -0,0 +1,10 @@
+#! /usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path('../../vendor/parser/lib', __FILE__))
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+require 'c_lexer'
+require 'parser/runner/ruby_parse'
+
+Parser::Runner::RubyParse.go(ARGV)
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'c_lexer'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/c_lexer.gemspec

@@ -0,0 +1,36 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'c_lexer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'c_lexer'
+  spec.version       = CLexer::VERSION
+  spec.authors       = ['Ilya Bylich']
+  spec.email         = ['ibylich@gmail.com']
+
+  spec.description   = %q{A Ruby parser written in C}
+  spec.summary       = spec.description
+  spec.homepage      = 'https://github.com/iliabylich/c_lexer'
+
+  spec.files         = `git ls-files`.split.reject do |f|
+    f.match(%r{^(test|spec|features|vendor)/})
+  end + ['ext/lexer/lexer.c']
+
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.extensions    = ['ext/lexer/extconf.rb']
+
+  spec.add_dependency             'ast',           '~> 2.4.0'
+  spec.add_dependency             'parser',        '= 2.5.1.0'
+
+  spec.add_development_dependency 'bundler',       '~> 1.16'
+  spec.add_development_dependency 'rake',          '~> 10.0'
+  spec.add_development_dependency 'rake-compiler', '~> 0.9'
+
+  # Parser dev dependencies
+  spec.add_development_dependency 'minitest',      '~> 5.10'
+  spec.add_development_dependency 'simplecov',     '~> 0.15.1'
+  spec.add_development_dependency 'racc',          '= 1.4.14'
+  spec.add_development_dependency 'cliver',        '~> 0.3.2'
+end

data/ext/lexer/cmdarg.h

@@ -0,0 +1,47 @@
+static VALUE lexer_push_cmdarg_state(VALUE self, VALUE bit)
+{
+  INIT_LEXER_STATE(self, state);
+  int bitval = RTEST(bit) ? 1 : 0;
+  stack_state_push(&state->cmdarg, bitval);
+  return Qnil;
+}
+
+static VALUE lexer_pop_cmdarg_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_pop(&state->cmdarg);
+  return Qnil;
+}
+
+static VALUE lexer_lexpop_cmdarg_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_lexpop(&state->cmdarg);
+  return Qnil;
+}
+
+static VALUE lexer_clear_cmdarg_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_clear(&state->cmdarg);
+  return Qnil;
+}
+
+static VALUE lexer_cmdarg_state_empty_p(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  return stack_state_empty_p(&state->cmdarg);
+}
+
+static VALUE lexer_cmdarg_state_value(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  return INT2NUM(stack_state_value(&state->cmdarg));
+}
+
+static VALUE lexer_set_cmdarg_state(VALUE self, VALUE value)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_set_value(&state->cmdarg, NUM2INT(value));
+  return Qtrue;
+}

data/ext/lexer/cond.h

@@ -0,0 +1,47 @@
+static VALUE lexer_push_cond_state(VALUE self, VALUE bit)
+{
+  INIT_LEXER_STATE(self, state);
+  int bitval = RTEST(bit) ? 1 : 0;
+  stack_state_push(&state->cond, bitval);
+  return Qnil;
+}
+
+static VALUE lexer_pop_cond_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_pop(&state->cond);
+  return Qnil;
+}
+
+static VALUE lexer_lexpop_cond_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_lexpop(&state->cond);
+  return Qnil;
+}
+
+static VALUE lexer_clear_cond_state(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_state_clear(&state->cond);
+  return Qnil;
+}
+
+static VALUE lexer_cond_state_empty_p(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  return stack_state_empty_p(&state->cond);
+}
+
+static VALUE lexer_cond_state_value(VALUE self)
+{
+  INIT_LEXER_STATE(self, state);
+  return INT2NUM(stack_state_value(&state->cond));
+}
+
+static VALUE lexer_set_cond_state(VALUE self, VALUE value)
+{
+  INIT_LEXER_STATE(self, state);
+  stack_set_value(&state->cond, NUM2INT(value));
+  return Qtrue;
+}

data/ext/lexer/emit_tables.h

@@ -0,0 +1,177 @@
+#define try_mapping(str, token) if (strcmp(value_str, str) == 0) { emit_token(state, token, value, start, end); return; }
+#define invalid_mapping rb_raise(rb_eArgError, "Invalid punctuation token: %s", value_str);
+
+static void emit_table_PUNCTUATION(lexer_state *state, VALUE value, long start, long end)
+{
+  const char *value_str = RSTRING_PTR(value);
+
+  try_mapping("=", tEQL);
+  try_mapping("&", tAMPER2);
+  try_mapping("|", tPIPE);
+  try_mapping("!", tBANG);
+  try_mapping("^", tCARET);
+  try_mapping("+", tPLUS);
+  try_mapping("-", tMINUS);
+  try_mapping("*", tSTAR2);
+  try_mapping("/", tDIVIDE);
+  try_mapping("%", tPERCENT);
+  try_mapping("~", tTILDE);
+  try_mapping(",", tCOMMA);
+  try_mapping(";", tSEMI);
+  try_mapping(".", tDOT);
+  try_mapping("..", tDOT2);
+  try_mapping("...", tDOT3);
+  try_mapping("[", tLBRACK2);
+  try_mapping("]", tRBRACK);
+  try_mapping("(", tLPAREN2);
+  try_mapping(")", tRPAREN);
+  try_mapping("?", tEH);
+  try_mapping(":", tCOLON);
+  try_mapping("&&", tANDOP);
+  try_mapping("||", tOROP);
+  try_mapping("-@", tUMINUS);
+  try_mapping("+@", tUPLUS);
+  try_mapping("~@", tTILDE);
+  try_mapping("**", tPOW);
+  try_mapping("->", tLAMBDA);
+  try_mapping("=~", tMATCH);
+  try_mapping("!~", tNMATCH);
+  try_mapping("==", tEQ);
+  try_mapping("!=", tNEQ);
+  try_mapping(">", tGT);
+  try_mapping(">>", tRSHFT);
+  try_mapping(">=", tGEQ);
+  try_mapping("<", tLT);
+  try_mapping("<<", tLSHFT);
+  try_mapping("<=", tLEQ);
+  try_mapping("=>", tASSOC);
+  try_mapping("::", tCOLON2);
+  try_mapping("===", tEQQ);
+  try_mapping("<=>", tCMP);
+  try_mapping("[]", tAREF);
+  try_mapping("[]=", tASET);
+  try_mapping("{", tLCURLY);
+  try_mapping("}", tRCURLY);
+  try_mapping("`", tBACK_REF2);
+  try_mapping("!@", tBANG);
+  try_mapping("&.", tANDDOT);
+
+  invalid_mapping;
+}
+
+static void emit_table_PUNCTUATION_BEGIN(lexer_state *state, VALUE value, long start, long end)
+{
+  const char *value_str = RSTRING_PTR(value);
+
+  try_mapping("&", tAMPER);
+  try_mapping("*", tSTAR);
+  try_mapping("**", tDSTAR);
+  try_mapping("+", tUPLUS);
+  try_mapping("-", tUMINUS);
+  try_mapping("::", tCOLON3);
+  try_mapping("(", tLPAREN);
+  try_mapping("{", tLBRACE);
+  try_mapping("[", tLBRACK);
+
+  invalid_mapping;
+}
+
+static void emit_table_KEYWORDS(lexer_state *state, VALUE value, long start, long end)
+{
+  const char *value_str = RSTRING_PTR(value);
+
+  try_mapping("if", kIF_MOD);
+  try_mapping("unless", kUNLESS_MOD);
+  try_mapping("while", kWHILE_MOD);
+  try_mapping("until", kUNTIL_MOD);
+  try_mapping("rescue", kRESCUE_MOD);
+  try_mapping("defined?", kDEFINED);
+  try_mapping("BEGIN", klBEGIN);
+  try_mapping("END", klEND);
+
+  try_mapping("class", kCLASS);
+  try_mapping("module", kMODULE);
+  try_mapping("def", kDEF);
+  try_mapping("undef", kUNDEF);
+  try_mapping("begin", kBEGIN);
+  try_mapping("end", kEND);
+  try_mapping("then", kTHEN);
+  try_mapping("elsif", kELSIF);
+  try_mapping("else", kELSE);
+  try_mapping("ensure", kENSURE);
+  try_mapping("case", kCASE);
+  try_mapping("when", kWHEN);
+  try_mapping("for", kFOR);
+  try_mapping("break", kBREAK);
+  try_mapping("next", kNEXT);
+  try_mapping("redo", kREDO);
+  try_mapping("retry", kRETRY);
+  try_mapping("in", kIN);
+  try_mapping("do", kDO);
+  try_mapping("return", kRETURN);
+  try_mapping("yield", kYIELD);
+  try_mapping("super", kSUPER);
+  try_mapping("self", kSELF);
+  try_mapping("nil", kNIL);
+  try_mapping("true", kTRUE);
+  try_mapping("false", kFALSE);
+  try_mapping("and", kAND);
+  try_mapping("or", kOR);
+  try_mapping("not", kNOT);
+  try_mapping("alias", kALIAS);
+  try_mapping("__FILE__", k__FILE__);
+  try_mapping("__LINE__", k__LINE__);
+  try_mapping("__ENCODING__", k__ENCODING__);
+
+  invalid_mapping;
+}
+
+static void emit_table_KEYWORDS_BEGIN(lexer_state *state, VALUE value, long start, long end)
+{
+  const char *value_str = RSTRING_PTR(value);
+
+  try_mapping("if", kIF);
+  try_mapping("unless", kUNLESS);
+  try_mapping("while", kWHILE);
+  try_mapping("until", kUNTIL);
+  try_mapping("rescue", kRESCUE);
+  try_mapping("defined?", kDEFINED);
+  try_mapping("BEGIN", klBEGIN);
+  try_mapping("END", klEND);
+
+  try_mapping("class", kCLASS);
+  try_mapping("module", kMODULE);
+  try_mapping("def", kDEF);
+  try_mapping("undef", kUNDEF);
+  try_mapping("begin", kBEGIN);
+  try_mapping("end", kEND);
+  try_mapping("then", kTHEN);
+  try_mapping("elsif", kELSIF);
+  try_mapping("else", kELSE);
+  try_mapping("ensure", kENSURE);
+  try_mapping("case", kCASE);
+  try_mapping("when", kWHEN);
+  try_mapping("for", kFOR);
+  try_mapping("break", kBREAK);
+  try_mapping("next", kNEXT);
+  try_mapping("redo", kREDO);
+  try_mapping("retry", kRETRY);
+  try_mapping("in", kIN);
+  try_mapping("do", kDO);
+  try_mapping("return", kRETURN);
+  try_mapping("yield", kYIELD);
+  try_mapping("super", kSUPER);
+  try_mapping("self", kSELF);
+  try_mapping("nil", kNIL);
+  try_mapping("true", kTRUE);
+  try_mapping("false", kFALSE);
+  try_mapping("and", kAND);
+  try_mapping("or", kOR);
+  try_mapping("not", kNOT);
+  try_mapping("alias", kALIAS);
+  try_mapping("__FILE__", k__FILE__);
+  try_mapping("__LINE__", k__LINE__);
+  try_mapping("__ENCODING__", k__ENCODING__);
+
+  invalid_mapping;
+}