code-lexer 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d6c98649f07e77d4148fb744db9b79fb0ed714113d6aaee7cf02f249868070c9
+  data.tar.gz: 6c6deb5e8f6778a036dd60cf49d649192620455d0197358b4783e0df0dd91bce
+SHA512:
+  metadata.gz: 60556343e374a1c7ea58a076473fa98e9a15f4b8a89451788675a2415eb7c2da05f9255463abe9a2ae75da239c84a6186f8d5bc7aa570dd16580bdd8e685a7e7
+  data.tar.gz: 974dcf39a0a41c496f61429dd0e5b16557a80be856fef450dddac29a19ca72dc4b4ed0c33a2ff9226a4c683af665b44d1b26c2b36ac49d15cd96d47cc350cd20

data/lib/code-lexer/abstractor.rb

@@ -0,0 +1,110 @@
+require_relative 'token'
+
+module CodeLexer
+    class Abstractor
+        attr_reader     :dictionary
+        
+        def initialize(dictionary=[])
+            @dictionary = ["NOOP"] + dictionary
+        end
+        
+        def abstract_identifiers
+            @abstract_identifiers = true
+            return self
+        end
+        
+        def abstract_numbers
+            @abstract_numbers = true
+            return self
+        end
+        
+        def abstract_comments
+            @abstract_comments = true
+            return self
+        end
+        
+        def abstract_strings
+            @abstract_strings = true
+            return self
+        end
+        
+        def abstract_spaces
+            @abstract_spaces = true
+            return self
+        end
+        
+        def remove_spaces
+            @remove_spaces = true
+            return self
+        end
+        
+        def remove_newlines
+            @remove_newlines = true
+            return self
+        end
+        
+        def remove_comments
+            @remove_comments = true
+            return self
+        end
+        
+        def abstract!(tokens)
+            if @abstract_identifiers
+                identifier_tokens = tokens.select { |t| t.type == :identifier }
+                identifiers = identifier_tokens.map { |id| id.value }.uniq
+                
+                identifiers.each do |id|
+                    if @dictionary.include?(id)
+                        abstracted_id = @dictionary.index(id)
+                    else
+                        abstracted_id = @dictionary.size
+                        @dictionary << id 
+                    end
+                        
+                    identifier_tokens.select { |t| t.value == id }.each do |matching_token|
+                        matching_token.abstracted_value = Token.special("ID#{abstracted_id}")
+                    end
+                end
+            end
+            
+            if @remove_comments
+                tokens.delete_if { |t| t.type == :comment }
+            elsif @abstract_comments
+                tokens.select { |t| t.type == :comment }.each do |comment_token|
+                    comment_token.abstracted_value = Token.special("COMMENT")
+                end
+            end
+            
+            if @abstract_numbers
+                tokens.select { |t| t.type == :number }.each do |number_token|
+                    number_token.abstracted_value = Token.special("NUMBER")
+                end
+            end
+            
+            if @abstract_strings
+                tokens.select { |t| t.type == :string }.each do |string_token|
+                    string_token.abstracted_value = Token.special("STRING")
+                end
+            end
+            
+            if @remove_newlines
+                tokens.delete_if { |t| t.type == :newline }
+            end
+            
+            if @remove_spaces
+                tokens.delete_if { |t| t.type == :space }
+            elsif @abstract_spaces
+                tokens.select { |t| t.type == :space }.each do |space_token|
+                    previous_index = tokens.index(space_token) - 1
+                    if previous_index < 0 || tokens[previous_index].type == :newline
+                        space_token.abstracted_value = Token.special("INDENTATION")
+                    else
+                        space_token.abstracted_value = Token.special("WHITESPACE")
+                    end
+                end
+            end
+            
+            return self
+        end
+    end
+end

data/lib/code-lexer/config.rb

@@ -0,0 +1,38 @@
+module CodeLexer
+    class Config
+        attr_reader     :rules
+        def initialize(path)
+            @config = File.basename(path)
+            @rules = []
+            
+            load_rules(File.read(path))
+        end
+        
+        def matching_rule(text)
+            min_score = 10000
+            min_couple = []
+            @rules.each do |name, regex|
+                if (score = (text =~ regex))
+                    if score < min_score
+                        min_score = score
+                        min_couple = [name, regex]
+                    end
+                end
+            end
+            
+            return *min_couple
+        end
+        
+        private
+        def load_rules(content)
+            content.split("\n").each do |line|
+                name, regex = line.split(":", 2)
+                regex = Regexp.new("^" + regex)
+                
+                @rules << [name.to_sym, regex]
+            end
+            
+            @rules << [:other, /./]
+        end
+    end 
+end

data/lib/code-lexer/languages/javascript.clex

@@ -0,0 +1,24 @@
+keyword:(?:abstract|arguments|boolean|break|byte|case|catch|char|const|continue|debugger|default|delete|do|double|else|eval|false|final|finally|float|for|function|goto|if|implements|in|instanceof|int|interface|let|long|native|new|null|package|private|protected|public|return|short|static|switch|synchronized|this|throw|throws|transient|true|try|typeof|var|void|volatile|while|with|yield|class|enum|export|extends|import|super|from)
+identifier:[$A-Za-z_][$A-Za-z0-9_]*
+comment:\/\/[^.]*[\n\r]
+comment:\/\/[^.]*$
+comment:\/\*([^*]|[\r\n]|(\*+([^*/]|[\r\n])))*\*+\/
+string:\"([^"]|\\\")*\"
+string:\'[^']*\'
+number:\-?[0-9]
+number:\-?[1-9][0-9]*
+number:\-?[0-9]*\.[0-9]
+number:\-?[0-9]*\.[0-9]e\-?[0-9]+
+number:\-?0[Xx][0-9A-Fa-f]+
+number:\-?0[0-7]+
+operator:(\<\=|\>\=|\=\=|\=\=\=|\!\=\=|\!\=)
+operator:(\&\&|\||\|\||\!)
+operator:(\=|\+\=|\-\=|\/\=|\*\=|\%\=)
+operator:(\&|\||\~|\^|\<\<|\>\>)
+operator:(\+|\-|\/|\*|\%|\+\+|\-\-)
+operator:(\.|\,|\:)
+operator:(\<|\>)
+parenthesis:(\(|\)|\[|\]|\{|\})
+semicolon:\;
+newline:[\n\r]
+space:\s+

data/lib/code-lexer/lexer.rb

@@ -0,0 +1,66 @@
+require_relative 'token'
+require_relative 'abstractor'
+require_relative 'config'
+
+module CodeLexer
+    class Lexer
+        def initialize(config_path_or_config)
+            if config_path_or_config.is_a?(Config)
+                @config = config_path_or_config
+            else
+                @config = Config.new(config_path_or_config)
+            end
+        end
+        
+        def lex(content)
+            content = content.clone
+            tokens = []
+            while content.length > 0
+                token_name, regex = @config.matching_rule(content)
+                content.sub!(regex) do |value|
+                    tokens << Token.new(token_name, value)
+                    ""
+                end
+            end
+            
+            return LexedContent.new(tokens)
+        end
+    end
+    
+    class LexedContent
+        attr_reader     :tokens
+        
+        def initialize(tokens)
+            @tokens = tokens
+        end
+                
+        def token_lines
+            result = []
+            current_line = []
+            @tokens.each do |t|
+                if t.type == :newline
+                    result << current_line
+                    current_line = []
+                else
+                    current_line << t
+                end
+            end
+            
+            result << current_line
+            result.delete_if { |line| line.empty? }
+            
+            return result
+        end
+        
+        def token_stream(abstractor = nil)
+            abstractor.abstract!(@tokens) if abstractor
+            
+            result = []
+            @tokens.each do |token|
+                result << token.abstracted_value
+            end
+            
+            return result.join(" ")
+        end
+    end
+end

data/lib/code-lexer/token.rb

@@ -0,0 +1,42 @@
+module CodeLexer
+    class Token
+        SPECIAL_TOKEN_OPEN  = "¬"
+        SPECIAL_TOKEN_CLOSE = "¬"
+        
+        def self.special(token)
+            "#{SPECIAL_TOKEN_OPEN}#{token}#{SPECIAL_TOKEN_CLOSE}"
+        end
+        
+        attr_accessor :type
+        attr_accessor :value
+        attr_accessor :abstracted_value
+        
+        def initialize(type, value)
+            @type = type
+            self.value = value
+        end
+        
+        def value=(v)
+            @value = v
+            if @type == :newline
+                @abstracted_value = Token.special("NEWLINE")
+            elsif v =~ /\s/
+                @abstracted_value = Token.special(v.gsub(/\s/, "·"))
+            else
+                @abstracted_value = v
+            end
+        end
+        
+        def to_s
+            if @abstracted_value != @value
+                return "<#@type:#{@value.inspect}:#{@abstracted_value.inspect}>"
+            else
+                return "<#@type:#{@value.inspect}>"
+            end
+        end
+                
+        def ==(oth)
+            @type == oth.type && @value == oth.value && @abstracted_value == oth.abstracted_value
+        end
+    end
+end

data/lib/code-lexer.rb

@@ -0,0 +1,10 @@
+require_relative 'code-lexer/config'
+require_relative 'code-lexer/abstractor'
+require_relative 'code-lexer/lexer'
+require_relative 'code-lexer/token'
+
+module CodeLexer
+    def self.get(language)
+        return Lexer.new("#{File.dirname(File.expand_path(__FILE__))}/code-lexer/languages/#{language}.clex")
+    end
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: code-lexer
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Simone Scalabrino
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: code-assertions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+description: Source code lexer configurable for any programming language that allows
+  to tokenize and abstract a given source file
+email: s.scalabrino9@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/code-lexer.rb
+- lib/code-lexer/abstractor.rb
+- lib/code-lexer/config.rb
+- lib/code-lexer/languages/javascript.clex
+- lib/code-lexer/lexer.rb
+- lib/code-lexer/token.rb
+homepage: https://github.com/intersimone999/code-lexer
+licenses:
+- GPL-3.0-only
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.29
+signing_key:
+specification_version: 4
+summary: Simple source code lexer
+test_files: []