code-lexer 0.1 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6c98649f07e77d4148fb744db9b79fb0ed714113d6aaee7cf02f249868070c9
-  data.tar.gz: 6c6deb5e8f6778a036dd60cf49d649192620455d0197358b4783e0df0dd91bce
+  metadata.gz: 6ce8480d1b7e07f708c5cf0134a7eb05a6e85c493bc5cf391db1d0f36b195768
+  data.tar.gz: ebfed8650052cba1280b1a6ad8178f6080d2cd6fe70626153bb29e5c8a62f6e2
 SHA512:
-  metadata.gz: 60556343e374a1c7ea58a076473fa98e9a15f4b8a89451788675a2415eb7c2da05f9255463abe9a2ae75da239c84a6186f8d5bc7aa570dd16580bdd8e685a7e7
-  data.tar.gz: 974dcf39a0a41c496f61429dd0e5b16557a80be856fef450dddac29a19ca72dc4b4ed0c33a2ff9226a4c683af665b44d1b26c2b36ac49d15cd96d47cc350cd20
+  metadata.gz: 30ce58ffe14a55397dfdc23c1ab4cdb73e0b1e25e4b3eaa2f829157a50dd60c0cf6475a0911ea6e28d4ab98343b17179b31064dca984bbec53ddd811d6259b75
+  data.tar.gz: b4373e903dd74fc372d36b32370a2f7f0106f019b9e04a9049341443014c15276f0035579c2e7b93f69d9a4978c45761a193321322a3253a5a77cafa0f86e7dc

data/lib/code-lexer/abstractor.rb

@@ -8,6 +8,16 @@ module CodeLexer
             @dictionary = ["NOOP"] + dictionary
         end
         
+        def abstract_everything
+            self.abstract_identifiers
+            self.abstract_numbers
+            self.abstract_comments
+            self.abstract_strings
+            self.abstract_spaces
+            
+            return self
+        end
+        
         def abstract_identifiers
             @abstract_identifiers = true
             return self

data/lib/code-lexer/config.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module CodeLexer
     class Config
         attr_reader     :rules
@@ -5,7 +7,7 @@ module CodeLexer
             @config = File.basename(path)
             @rules = []
             
-            load_rules(File.read(path))
+            load_rules(path)
         end
         
         def matching_rule(text)
@@ -25,11 +27,15 @@ module CodeLexer
         
         private
         def load_rules(content)
-            content.split("\n").each do |line|
-                name, regex = line.split(":", 2)
-                regex = Regexp.new("^" + regex)
-                
-                @rules << [name.to_sym, regex]
+            parsed = YAML.load_file(content)
+            
+            
+            parsed['lexer'].each do |name, regexs|
+                regexs.each do |regex|
+                    p regex
+                    regex = Regexp.new("^" + regex)
+                    @rules << [name.to_sym, regex]
+                end
             end
             
             @rules << [:other, /./]

data/lib/code-lexer/languages/javascript.yml

@@ -0,0 +1,36 @@
+lexer:
+    keyword:
+        - (?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
+    identifier:
+        - "[$A-Za-z_][$A-Za-z0-9_]*"
+    comment:
+        - \/\/[^\n\r]*[\n\r]
+        - \/\/.*$
+        - \/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
+    string:
+        - \"([^"]|\\\")*\"
+        - \'[^']*\'
+    number:
+        - \-?[0-9]
+        - \-?[1-9][0-9]*
+        - \-?[0-9]*\.[0-9]
+        - \-?[0-9]*\.[0-9]e\-?[0-9]+
+        - \-?0[Xx][0-9A-Fa-f]+
+        - \-?0[0-7]+
+    operator:
+        - (\=\=\=|\!\=\=)
+        - (\<\=|\>\=|\=\=|\!\=)
+        - (\&\&|\|\||\!)
+        - (\+\=|\-\=|\/\=|\*\=|\%\=|\+\+|\-\-)
+        - (\&|\||\~|\^|\<\<|\>\>)
+        - (\=|\+|\-|\/|\*|\%)
+        - (\.|\,|\:)
+        - (\<|\>)
+    parenthesis:
+        - (\(|\)|\[|\]|\{|\})
+    semicolon:
+        - \;
+    newline:
+        - "[\\n\\r]"
+    space:
+        - \s+

data/lib/code-lexer.rb

@@ -5,6 +5,6 @@ require_relative 'code-lexer/token'
 
 module CodeLexer
     def self.get(language)
-        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.clex")
+        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.yml")
     end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: code-lexer
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: 0.5.1
 platform: ruby
 authors:
 - Simone Scalabrino
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-28 00:00:00.000000000 Z
+date: 2021-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -40,14 +40,14 @@ files:
 - lib/code-lexer.rb
 - lib/code-lexer/abstractor.rb
 - lib/code-lexer/config.rb
-- lib/code-lexer/languages/javascript.clex
+- lib/code-lexer/languages/javascript.yml
 - lib/code-lexer/lexer.rb
 - lib/code-lexer/token.rb
 homepage: https://github.com/intersimone999/code-lexer
 licenses:
 - GPL-3.0-only
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -62,8 +62,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
-signing_key:
+rubygems_version: 3.2.32
+signing_key: 
 specification_version: 4
 summary: Simple source code lexer
 test_files: []

data/lib/code-lexer/languages/javascript.clex

@@ -1,24 +0,0 @@
-keyword:(?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
-identifier:[$A-Za-z_][$A-Za-z0-9_]*
-comment:\/\/[^.]*[\n\r]
-comment:\/\/[^.]*$
-comment:\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
-string:\"([^"]|\\\")*\"
-string:\'[^']*\'
-number:\-?[0-9]
-number:\-?[1-9][0-9]*
-number:\-?[0-9]*\.[0-9]
-number:\-?[0-9]*\.[0-9]e\-?[0-9]+
-number:\-?0[Xx][0-9A-Fa-f]+
-number:\-?0[0-7]+
-operator:(\<\=|\>\=|\=\=|\=\=\=|\!\=\=|\!\=)
-operator:(\&\&|\||\|\||\!)
-operator:(\=|\+\=|\-\=|\/\=|\*\=|\%\=)
-operator:(\&|\||\~|\^|\<\<|\>\>)
-operator:(\+|\-|\/|\*|\%|\+\+|\-\-)
-operator:(\.|\,|\:)
-operator:(\<|\>)
-parenthesis:(\(|\)|\[|\]|\{|\})
-semicolon:\;
-newline:[\n\r]
-space:\s+