code-lexer 0.2 → 0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e953d35e258951fec7845c7e78b19918733e9c531db7b4bbf075008e9c37d1d
-  data.tar.gz: eea84826ff95b4ff9ea49aac52e2777968168485e2ea5205a73f9197550dc5cf
+  metadata.gz: 0be86a493e60a21bc3c8d16598c6f6af914a2521255bb604b01ffa9410ff399f
+  data.tar.gz: 46e60be81ceda1cc9621ef7519189a45cceaaa54afa27313a5c6d4c8be0e8377
 SHA512:
-  metadata.gz: a731c6149fc98518ef8c36963794173c3463b4626b15513317a46e0a933c6e734ad0ee749d4631505a2ee5a4524d600f0a4901a825b132d9f89ec148f0c6d138
-  data.tar.gz: a8ad70dc8c4eb7189bb84d153c9bc6c99dc709ab28a0295074c596322190392b72d2205739351850786a46faee6d6d99ec85a1d4dcbc60d928c311ee9ca3d1a8
+  metadata.gz: adcc4ee93e5c58da53df775584dbd4de485d8af7af232e2a7aa379a1ec09c1e928a804139983d585773f3b837bdb6f9fe95623107dfceaa0fb3afc2ddd5a7593
+  data.tar.gz: e9ee16aa762e15515d58930b29b5c8f784baee483f40fa1ae520734a0efc98f7680ffa5a4f6c88de3c261b836e17bb9e529aa4adc55314928d3a60f7f0d0d2da

data/lib/code-lexer/config.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module CodeLexer
     class Config
         attr_reader     :rules
@@ -5,7 +7,7 @@ module CodeLexer
             @config = File.basename(path)
             @rules = []
             
-            load_rules(File.read(path))
+            load_rules(path)
         end
         
         def matching_rule(text)
@@ -25,11 +27,15 @@ module CodeLexer
         
         private
         def load_rules(content)
-            content.split("\n").each do |line|
-                name, regex = line.split(":", 2)
-                regex = Regexp.new("^" + regex)
-                
-                @rules << [name.to_sym, regex]
+            parsed = YAML.load_file(content)
+            
+            
+            parsed['lexer'].each do |name, regexs|
+                regexs.each do |regex|
+                    p regex
+                    regex = Regexp.new("^" + regex)
+                    @rules << [name.to_sym, regex]
+                end
             end
             
             @rules << [:other, /./]

data/lib/code-lexer/languages/javascript.yml

@@ -0,0 +1,36 @@
+lexer:
+    keyword:
+        - (?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
+    identifier:
+        - "[$A-Za-z_][$A-Za-z0-9_]*"
+    comment:
+        - \/\/[^\n\r]*[\n\r]
+        - \/\/.*$
+        - \/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
+    string:
+        - \"([^"]|\\\")*\"
+        - \'[^']*\'
+    number:
+        - \-?[0-9]*\.[0-9]e\-?[0-9]+
+        - \-?[0-9]*\.[0-9]
+        - \-?[1-9][0-9]*
+        - \-?0[Xx][0-9A-Fa-f]+
+        - \-?[0-9]
+        - \-?0[0-7]+
+    operator:
+        - (\=\=\=|\!\=\=)
+        - (\<\=|\>\=|\=\=|\!\=)
+        - (\&\&|\|\||\!)
+        - (\+\=|\-\=|\/\=|\*\=|\%\=|\+\+|\-\-)
+        - (\&|\||\~|\^|\<\<|\>\>)
+        - (\=|\+|\-|\/|\*|\%)
+        - (\.|\,|\:)
+        - (\<|\>)
+    parenthesis:
+        - (\(|\)|\[|\]|\{|\})
+    semicolon:
+        - \;
+    newline:
+        - "[\\n\\r]"
+    space:
+        - \s+

data/lib/code-lexer.rb

@@ -5,6 +5,6 @@ require_relative 'code-lexer/token'
 
 module CodeLexer
     def self.get(language)
-        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.clex")
+        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.yml")
     end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: code-lexer
 version: !ruby/object:Gem::Version
-  version: '0.2'
+  version: '0.6'
 platform: ruby
 authors:
 - Simone Scalabrino
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-28 00:00:00.000000000 Z
+date: 2021-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: code-assertions
@@ -40,7 +40,7 @@ files:
 - lib/code-lexer.rb
 - lib/code-lexer/abstractor.rb
 - lib/code-lexer/config.rb
-- lib/code-lexer/languages/javascript.clex
+- lib/code-lexer/languages/javascript.yml
 - lib/code-lexer/lexer.rb
 - lib/code-lexer/token.rb
 homepage: https://github.com/intersimone999/code-lexer

data/lib/code-lexer/languages/javascript.clex

@@ -1,24 +0,0 @@
-keyword:(?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
-identifier:[$A-Za-z_][$A-Za-z0-9_]*
-comment:\/\/[^.]*[\n\r]
-comment:\/\/[^.]*$
-comment:\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
-string:\"([^"]|\\\")*\"
-string:\'[^']*\'
-number:\-?[0-9]
-number:\-?[1-9][0-9]*
-number:\-?[0-9]*\.[0-9]
-number:\-?[0-9]*\.[0-9]e\-?[0-9]+
-number:\-?0[Xx][0-9A-Fa-f]+
-number:\-?0[0-7]+
-operator:(\<\=|\>\=|\=\=|\=\=\=|\!\=\=|\!\=)
-operator:(\&\&|\||\|\||\!)
-operator:(\=|\+\=|\-\=|\/\=|\*\=|\%\=)
-operator:(\&|\||\~|\^|\<\<|\>\>)
-operator:(\+|\-|\/|\*|\%|\+\+|\-\-)
-operator:(\.|\,|\:)
-operator:(\<|\>)
-parenthesis:(\(|\)|\[|\]|\{|\})
-semicolon:\;
-newline:[\n\r]
-space:\s+