grnexus 1.0.2

data/lib/grnexus_text_proccessing.rb

@@ -0,0 +1,295 @@
+# frozen_string_literal: true
+
+require 'ffi'
+
+module GRNexusTextProcessing
+  extend FFI::Library
+
+  # Detect and load appropriate library
+  def self.detect_library
+    script_dir = File.dirname(File.expand_path(__FILE__))
+    case RbConfig::CONFIG['host_os']
+    when /mswin|mingw|cygwin/
+      File.join(script_dir, '..', 'exports', 'Windows', 'text_processing.dll')
+    when /darwin/
+      File.join(script_dir, '..', 'exports', 'Mac', 'text_processing.dylib')
+    when /linux/
+      File.join(script_dir, '..', 'exports', 'Linux', 'text_processing.so')
+    else
+      raise "Sistema operativo no soportado: #{RbConfig::CONFIG['host_os']}"
+    end
+  end
+
+  ffi_lib detect_library
+
+  # Define GRNexusData structure
+  class GRNexusData < FFI::Struct
+    layout :data, :pointer,
+           :type, :int,
+           :size, :size_t,
+           :stride, :size_t,
+           :dims, [:size_t, 3]
+  end
+
+  # Define GRNexusVocabulary structure
+  class GRNexusVocabulary < FFI::Struct
+    layout :tokens, :pointer,
+           :indices, :pointer,
+           :frequencies, :pointer,
+           :idf_scores, :pointer,
+           :vocab_size, :size_t,
+           :max_token_len, :size_t,
+           :total_tokens, :size_t,
+           :special_tokens, [:pointer, 10],
+           :special_indices, :pointer
+  end
+
+  # Define GRNexusEmbeddings structure
+  class GRNexusEmbeddings < FFI::Struct
+    layout :embeddings, :pointer,
+           :vocab_size, :size_t,
+           :embedding_dim, :size_t,
+           :norms, :pointer
+  end
+
+  # Define GRNexusTFIDF structure
+  class GRNexusTFIDF < FFI::Struct
+    layout :tfidf_matrix, :pointer,
+           :num_docs, :size_t,
+           :vocab_size, :size_t
+  end
+
+  # Function definitions
+  attach_function :create_vocabulary_advanced, [:pointer, :size_t, :size_t], :pointer
+  attach_function :normalize_vocabulary_advanced, [:string, :pointer, :pointer, :size_t, :string], :int
+  attach_function :denormalize_vocabulary, [:pointer, :pointer, :pointer], :int
+  attach_function :vectorize_text_tfidf, [:string, :pointer, :pointer, :pointer], :int
+  attach_function :create_embeddings, [:pointer, :size_t], :pointer
+  attach_function :find_similar_embeddings, [:pointer, :size_t, :pointer, :pointer, :size_t], :int
+  attach_function :create_tfidf_model, [:pointer, :pointer, :size_t], :pointer
+  attach_function :find_similar_documents, [:pointer, :size_t, :pointer, :pointer, :size_t], :int
+  attach_function :free_vocabulary, [:pointer], :void
+  attach_function :free_embeddings, [:pointer], :void
+  attach_function :free_tfidf, [:pointer], :void
+
+  # Vocabulary class
+  class Vocabulary
+    attr_reader :vocab_ptr
+
+    def initialize(documents, max_vocab_size: 50000)
+      # Convert Ruby strings to C strings
+      c_docs = FFI::MemoryPointer.new(:pointer, documents.size)
+      documents.each_with_index do |doc, i|
+        c_docs[i].put_pointer(0, FFI::MemoryPointer.from_string(doc))
+      end
+
+      @vocab_ptr = GRNexusTextProcessing.create_vocabulary_advanced(c_docs, documents.size, max_vocab_size)
+      raise 'Failed to create vocabulary' if @vocab_ptr.null?
+
+      @vocab = GRNexusVocabulary.new(@vocab_ptr)
+      ObjectSpace.define_finalizer(self, self.class.finalize(@vocab_ptr))
+    end
+
+    def self.finalize(vocab_ptr)
+      proc { GRNexusTextProcessing.free_vocabulary(vocab_ptr) unless vocab_ptr.null? }
+    end
+
+    def size
+      @vocab[:vocab_size]
+    end
+
+    def total_tokens
+      @vocab[:total_tokens]
+    end
+
+    def get_tokens(max_tokens: 100)
+      tokens = []
+      tokens_ptr = @vocab[:tokens]
+      [max_tokens, size].min.times do |i|
+        token_ptr = tokens_ptr.get_pointer(i * FFI::Pointer.size)
+        tokens << token_ptr.read_string unless token_ptr.null?
+      end
+      tokens
+    end
+
+    def normalize_text(text, max_length: 512, strategy: 'pad_right')
+      output_data = GRNexusData.new
+      output_buffer = FFI::MemoryPointer.new(:double, max_length)
+      output_data[:data] = output_buffer
+      output_data[:type] = 1
+      output_data[:size] = max_length
+      output_data[:stride] = 1
+      output_data[:dims][0] = max_length
+      output_data[:dims][1] = 0
+      output_data[:dims][2] = 0
+
+      result = GRNexusTextProcessing.normalize_vocabulary_advanced(
+        text,
+        @vocab_ptr,
+        output_data.pointer,
+        max_length,
+        strategy
+      )
+
+      raise "Text normalization failed with code: #{result}" if result != 0
+
+      output_buffer.read_array_of_double(max_length)
+    end
+
+    def denormalize_indices(indices)
+      input_data = GRNexusData.new
+      input_buffer = FFI::MemoryPointer.new(:double, indices.size)
+      input_buffer.write_array_of_double(indices)
+      input_data[:data] = input_buffer
+      input_data[:type] = 1
+      input_data[:size] = indices.size
+      input_data[:stride] = 1
+      input_data[:dims][0] = indices.size
+      input_data[:dims][1] = 0
+      input_data[:dims][2] = 0
+
+      output_text_ptr = FFI::MemoryPointer.new(:pointer)
+
+      result = GRNexusTextProcessing.denormalize_vocabulary(
+        input_data.pointer,
+        @vocab_ptr,
+        output_text_ptr
+      )
+
+      raise "Denormalization failed with code: #{result}" if result != 0
+
+      text = output_text_ptr.read_pointer.read_string
+      text
+    end
+  end
+
+  # TextVectorizer class
+  class TextVectorizer
+    def initialize(vocabulary)
+      @vocabulary = vocabulary
+    end
+
+    def vectorize(text)
+      output_data = GRNexusData.new
+      output_buffer = FFI::MemoryPointer.new(:double, @vocabulary.size)
+      output_data[:data] = output_buffer
+      output_data[:type] = 1
+      output_data[:size] = @vocabulary.size
+      output_data[:stride] = 1
+      output_data[:dims][0] = @vocabulary.size
+      output_data[:dims][1] = 0
+      output_data[:dims][2] = 0
+
+      result = GRNexusTextProcessing.vectorize_text_tfidf(
+        text,
+        @vocabulary.vocab_ptr,
+        output_data.pointer,
+        nil
+      )
+
+      raise "Vectorization failed with code: #{result}" if result != 0
+
+      output_buffer.read_array_of_double(@vocabulary.size)
+    end
+  end
+
+  # TextEmbeddings class
+  class TextEmbeddings
+    def initialize(vocabulary, embedding_dim: 100)
+      @vocabulary = vocabulary
+      @embedding_dim = embedding_dim
+      @embeddings_ptr = GRNexusTextProcessing.create_embeddings(vocabulary.vocab_ptr, embedding_dim)
+
+      raise 'Failed to create embeddings' if @embeddings_ptr.null?
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@embeddings_ptr))
+    end
+
+    def self.finalize(embeddings_ptr)
+      proc { GRNexusTextProcessing.free_embeddings(embeddings_ptr) unless embeddings_ptr.null? }
+    end
+
+    def find_similar(token_idx, top_k: 10)
+      similarities = FFI::MemoryPointer.new(:double, @vocabulary.size)
+      similar_indices = FFI::MemoryPointer.new(:size_t, top_k)
+
+      result = GRNexusTextProcessing.find_similar_embeddings(
+        @embeddings_ptr,
+        token_idx,
+        similarities,
+        similar_indices,
+        top_k
+      )
+
+      raise "Finding similar embeddings failed with code: #{result}" if result != 0
+
+      # Read size_t array manually (size_t is typically ulong)
+      indices = []
+      top_k.times do |i|
+        indices << similar_indices.get_ulong(i * FFI.type_size(:size_t))
+      end
+      sims = indices.map { |i| similarities.get_double(i * 8) }
+
+      [indices, sims]
+    end
+  end
+
+  # TFIDFModel class
+  class TFIDFModel
+    def initialize(vocabulary, documents)
+      @vocabulary = vocabulary
+      @num_docs = documents.size
+
+      # Convert documents to C strings
+      c_docs = FFI::MemoryPointer.new(:pointer, documents.size)
+      documents.each_with_index do |doc, i|
+        c_docs[i].put_pointer(0, FFI::MemoryPointer.from_string(doc))
+      end
+
+      @tfidf_ptr = GRNexusTextProcessing.create_tfidf_model(vocabulary.vocab_ptr, c_docs, documents.size)
+
+      raise 'Failed to create TF-IDF model' if @tfidf_ptr.null?
+
+      ObjectSpace.define_finalizer(self, self.class.finalize(@tfidf_ptr))
+    end
+
+    def self.finalize(tfidf_ptr)
+      proc { GRNexusTextProcessing.free_tfidf(tfidf_ptr) unless tfidf_ptr.null? }
+    end
+
+    def find_similar_documents(doc_idx, top_k: 5)
+      similarities = FFI::MemoryPointer.new(:double, @num_docs)
+      similar_docs = FFI::MemoryPointer.new(:size_t, top_k)
+
+      result = GRNexusTextProcessing.find_similar_documents(
+        @tfidf_ptr,
+        doc_idx,
+        similarities,
+        similar_docs,
+        top_k
+      )
+
+      raise "Finding similar documents failed with code: #{result}" if result != 0
+
+      # Read size_t array manually (size_t is typically ulong)
+      indices = []
+      top_k.times do |i|
+        indices << similar_docs.get_ulong(i * FFI.type_size(:size_t))
+      end
+      sims = indices.map { |i| similarities.get_double(i * 8) }
+
+      [indices, sims]
+    end
+  end
+
+  # Legacy placeholder classes for compatibility
+  class TextProcessor
+    def process(text)
+      raise NotImplementedError, 'Debes implementar el método de procesamiento de texto'
+    end
+  end
+
+  class Tokenizer < TextProcessor; end
+  class WordEmbedding < TextProcessor; end
+  class TextNormalization < TextProcessor; end
+end

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: grnexus
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- GR Code Digital Solutions
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: "    GRNexus is a revolutionary cross-language AI platform that combines
+  the elegance \n    of Ruby with the raw power of native C acceleration. Train models
+  in Ruby and \n    deploy them in Python (or vice versa) with full compatibility.
+  Features include \n    35+ activation functions, 12+ layer types, complete NLP pipeline,
+  40+ numeric \n    operations, and intelligent training callbacks. 10-100x faster
+  than pure Ruby \n    implementations thanks to native C core.\n"
+email:
+- gr.code.studio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.md
+files:
+- LICENSE
+- README.md
+- exports/Linux/libgrnexus.so
+- exports/Mac/activations.dylib
+- exports/Mac/grnexus_core.dylib
+- exports/Mac/machine_learning.dylib
+- exports/Mac/normalization.dylib
+- exports/Mac/numeric_proccessing.dylib
+- exports/Mac/text_processing.dylib
+- exports/Windows/activations.dll
+- exports/Windows/grnexus_core.dll
+- exports/Windows/machine_learning.dll
+- exports/Windows/normalization.dll
+- exports/Windows/numeric_proccessing.dll
+- exports/Windows/text_processing.dll
+- lib/grnexus.rb
+- lib/grnexus_activations.rb
+- lib/grnexus_callbacks.rb
+- lib/grnexus_core.rb
+- lib/grnexus_layers.rb
+- lib/grnexus_machine_learning.rb
+- lib/grnexus_normalization.rb
+- lib/grnexus_numeric_proccessing.rb
+- lib/grnexus_text_proccessing.rb
+homepage: https://github.com/grcodedigitalsolutions/GRNexus
+licenses:
+- GPL-3.0-or-later
+metadata:
+  homepage_uri: https://github.com/grcodedigitalsolutions/GRNexus
+  bug_tracker_uri: https://github.com/grcodedigitalsolutions/GRNexus/issues
+  documentation_uri: https://github.com/grcodedigitalsolutions/GRNexus/blob/main/ruby/README.md
+  source_code_uri: https://github.com/grcodedigitalsolutions/GRNexus/tree/main/ruby
+  changelog_uri: https://github.com/grcodedigitalsolutions/GRNexus/blob/main/CHANGELOG.md
+post_install_message: "    \n                                                                   \n
+  \      GRNexus v1.0.2 installed successfully!                  \n                                                                   \n
+  \      The Ultimate Cross-Language Neural Network Framework       \n                                                                   \n
+  \      Features:                                                   \n       • 35+
+  Activation Functions (GELU, Swish, Mish, etc.)      \n       • 12+ Layer Types (Dense,
+  Conv2D, LSTM, GRU, etc.)        \n       • Complete NLP Pipeline (Tokenization,
+  Embeddings, etc.)   \n       • 40+ Numeric Operations                                   \n
+  \      • Cross-Language Model Compatibility (Ruby ↔ Python)       \n       • Native
+  C Acceleration (10-100x faster)                   \n                                                                   \n
+  \        IMPORTANT: Native Libraries Required                   \n       This gem
+  uses FFI to call native C libraries for           \n       performance. Pre-compiled
+  binaries are included for:       \n       • Linux (.so)                                              \n
+  \      • macOS (.dylib)                                           \n       • Windows
+  (.dll)                                           \n                                                                   \n
+  \      Get started:                                                \n       require
+  'grnexus'                                          \n                                                                   \n
+  \      Documentation: https://github.com/grcodedigitalsolutions/GRNexus \n\n    \n"
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.2
+specification_version: 4
+summary: High-performance cross-language neural network framework
+test_files: []