code-lexer 0.4 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13a18bbab7eb8a08deb270c5213c34b0a21d3ada73414d0cf7bfdd35606e4388
-  data.tar.gz: f22c5fb83d5521dae653992f17de00a18fcc9a6fef37f026ab8b1051b7085fd0
+  metadata.gz: 6ce8480d1b7e07f708c5cf0134a7eb05a6e85c493bc5cf391db1d0f36b195768
+  data.tar.gz: ebfed8650052cba1280b1a6ad8178f6080d2cd6fe70626153bb29e5c8a62f6e2
 SHA512:
-  metadata.gz: ccb2e6926b6868f3f4a9bdb18b420157dfc09beace098d9eceaddc85baffe2d2d63dabc2de6256bd1bdb0dc334adc16f36c60238700664c119a5327b01dd54a5
-  data.tar.gz: fb1d5bca13e9c23cba238ca9f85839a92238789fab85b5530408c6ddcd8e4f58bcebd8c000f2107b65ffe8b12783583684393c23f6d6581f787e46ede62855ff
+  metadata.gz: 30ce58ffe14a55397dfdc23c1ab4cdb73e0b1e25e4b3eaa2f829157a50dd60c0cf6475a0911ea6e28d4ab98343b17179b31064dca984bbec53ddd811d6259b75
+  data.tar.gz: b4373e903dd74fc372d36b32370a2f7f0106f019b9e04a9049341443014c15276f0035579c2e7b93f69d9a4978c45761a193321322a3253a5a77cafa0f86e7dc

data/lib/code-lexer/config.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module CodeLexer
     class Config
         attr_reader     :rules
@@ -5,7 +7,7 @@ module CodeLexer
             @config = File.basename(path)
             @rules = []
             
-            load_rules(File.read(path))
+            load_rules(path)
         end
         
         def matching_rule(text)
@@ -25,11 +27,15 @@ module CodeLexer
         
         private
         def load_rules(content)
-            content.split("\n").each do |line|
-                name, regex = line.split(":", 2)
-                regex = Regexp.new("^" + regex)
-                
-                @rules << [name.to_sym, regex]
+            parsed = YAML.load_file(content)
+            
+            
+            parsed['lexer'].each do |name, regexs|
+                regexs.each do |regex|
+                    p regex
+                    regex = Regexp.new("^" + regex)
+                    @rules << [name.to_sym, regex]
+                end
             end
             
             @rules << [:other, /./]

data/lib/code-lexer/languages/javascript.yml

@@ -0,0 +1,36 @@
+lexer:
+    keyword:
+        - (?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
+    identifier:
+        - "[$A-Za-z_][$A-Za-z0-9_]*"
+    comment:
+        - \/\/[^\n\r]*[\n\r]
+        - \/\/.*$
+        - \/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
+    string:
+        - \"([^"]|\\\")*\"
+        - \'[^']*\'
+    number:
+        - \-?[0-9]
+        - \-?[1-9][0-9]*
+        - \-?[0-9]*\.[0-9]
+        - \-?[0-9]*\.[0-9]e\-?[0-9]+
+        - \-?0[Xx][0-9A-Fa-f]+
+        - \-?0[0-7]+
+    operator:
+        - (\=\=\=|\!\=\=)
+        - (\<\=|\>\=|\=\=|\!\=)
+        - (\&\&|\|\||\!)
+        - (\+\=|\-\=|\/\=|\*\=|\%\=|\+\+|\-\-)
+        - (\&|\||\~|\^|\<\<|\>\>)
+        - (\=|\+|\-|\/|\*|\%)
+        - (\.|\,|\:)
+        - (\<|\>)
+    parenthesis:
+        - (\(|\)|\[|\]|\{|\})
+    semicolon:
+        - \;
+    newline:
+        - "[\\n\\r]"
+    space:
+        - \s+

data/lib/code-lexer.rb

@@ -5,6 +5,6 @@ require_relative 'code-lexer/token'
 
 module CodeLexer
     def self.get(language)
-        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.clex")
+        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.yml")
     end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: code-lexer
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: 0.5.1
 platform: ruby
 authors:
 - Simone Scalabrino
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-28 00:00:00.000000000 Z
+date: 2021-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -40,7 +40,7 @@ files:
 - lib/code-lexer.rb
 - lib/code-lexer/abstractor.rb
 - lib/code-lexer/config.rb
-- lib/code-lexer/languages/javascript.clex
+- lib/code-lexer/languages/javascript.yml
 - lib/code-lexer/lexer.rb
 - lib/code-lexer/token.rb
 homepage: https://github.com/intersimone999/code-lexer

data/lib/code-lexer/languages/javascript.clex

@@ -1,25 +0,0 @@
-keyword:(?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
-identifier:[$A-Za-z_][$A-Za-z0-9_]*
-comment:\/\/[^\n\r]*[\n\r]
-comment:\/\/.*$
-comment:\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
-string:\"([^"]|\\\")*\"
-string:\'[^']*\'
-number:\-?[0-9]
-number:\-?[1-9][0-9]*
-number:\-?[0-9]*\.[0-9]
-number:\-?[0-9]*\.[0-9]e\-?[0-9]+
-number:\-?0[Xx][0-9A-Fa-f]+
-number:\-?0[0-7]+
-operator:(\=\=\=|\!\=\=)
-operator:(\<\=|\>\=|\=\=|\!\=)
-operator:(\&\&|\|\||\!)
-operator:(\+\=|\-\=|\/\=|\*\=|\%\=|\+\+|\-\-)
-operator:(\&|\||\~|\^|\<\<|\>\>)
-operator:(\=|\+|\-|\/|\*|\%)
-operator:(\.|\,|\:)
-operator:(\<|\>)
-parenthesis:(\(|\)|\[|\]|\{|\})
-semicolon:\;
-newline:[\n\r]
-space:\s+