klue-langcraft 0.1.1 → 0.3.0

data/lib/base_process.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# file: lib/base_process.rb
+
+class BaseProcess
+  attr_reader :key
+
+  def initialize(key)
+    @key = key
+  end
+
+  def deep_match(input, predicate)
+    matches = []
+
+    # If the current input is a Hash, iterate over each key-value pair
+    if input.is_a?(Hash)
+
+      input.each do |key, value|
+
+        # If the value matches the predicate, add it to matches
+        if predicate.call(key, value)
+          matches << value
+        end
+
+        # Continue searching deeper within the value
+        matches.concat(deep_match(value, predicate))
+      end
+
+    # If the input is an Array, iterate over each item
+    elsif input.is_a?(Array)
+
+      input.each do |item|
+
+        # Continue searching within each item of the array
+        matches.concat(deep_match(item, predicate))
+      end
+    end
+
+    matches
+  end
+end

data/lib/dsl_folder_watcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class DSLFolderWatcher
+  def self.watch(folder_path)
+    puts "Watching: #{folder_path}"
+    listener = Listen.to(folder_path) do |modified, added, _removed|
+      changes = (modified + added).uniq
+
+      # DEBOUNCE CURRENTLY NOT WORKING
+      # debounce_map = {}
+      # debounce_interval = 1 # seconds
+
+      changes.each do |file_path|
+        next unless File.extname(file_path) == '.klue'
+
+        puts file_path
+
+        # debounce_map[file_path] ||= Time.now
+        # next unless Time.now - debounce_map[file_path] >= debounce_interval
+
+        # debounce_map[file_path] = Time.now
+
+        base_name = file_path.gsub(/\.klue$/, '')
+        input_file = "#{base_name}.klue"
+        output_file = "#{base_name}.json"
+
+        interpreter = DSLInterpreter.new
+        if interpreter.process('', input_file, output_file)
+          # Process the JSON data to add 'process-data' details
+          dsl_processor = DSLProcessData.new
+          dsl_processor.process('', output_file, output_file)
+          # SKIP EXTEND FILE FOR NOW AND REWRITE THE OUTPUTFILE
+          # dsl_processor.process('', output_file, extended_output_file)
+
+          # interpreter.send_to_endpoint
+        else
+          puts 'Skipping further processing due to errors in DSL interpretation.'
+        end
+      end
+
+      # Remove old entries from debounce_map to prevent memory bloat
+      # debounce_map.each_key do |key|
+      #   debounce_map.delete(key) if Time.now - debounce_map[key] > debounce_interval * 2
+      # end
+    end
+    listener.start
+    puts "Wait for changes: #{folder_path}"
+    sleep
+  end
+end

data/lib/dsl_interpreter.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/67064770-d524-8002-8344-3091e895d150
+# - https://chatgpt.com/c/6706289c-9b9c-8002-86e3-f9198c1c608a
+# - https://chatgpt.com/c/670dcd34-5dbc-8002-ad7a-d4df54a6a2e0
+#
+class DSLInterpreter
+  def initialize
+    @data = {}
+  end
+
+  # Capturing top-level DSL methods
+  def method_missing(method_name, *args, &block)
+    key = method_name
+    value = process_args(args, block)
+
+    # Append key-value to the current context of @data
+    if @data[key]
+      @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+      @data[key] << value
+    else
+      @data[key] = value
+    end
+  end
+
+  # A method to handle parameters and nested blocks
+  def process_args(args, block)
+    data = {}
+
+    # Handling positional and named parameters separately
+    positional_args = []
+    named_args = {}
+
+    args.each do |arg|
+      if arg.is_a?(Hash)
+        named_args.merge!(arg)
+      else
+        positional_args << arg
+      end
+    end
+
+    # Assign positional parameters generically
+    positional_args.each_with_index do |arg, index|
+      data[:"param#{index + 1}"] = arg
+    end
+
+    # Merge named parameters directly
+    data.merge!(named_args)
+
+    # Handling a nested block
+    if block
+      interpreter = DSLInterpreter.new
+      interpreter.instance_eval(&block)
+      data.merge!(interpreter.data)
+    end
+
+    data.empty? ? nil : data
+  end
+
+  # To access data after interpreting
+  attr_reader :data
+
+  # Reading file and evaluating as Ruby
+  def process(base_path, input_file, output_file)
+    file_path = File.join(base_path, input_file)
+    content = File.read(file_path)
+
+    # begin
+    instance_eval(content)
+    # rescue SyntaxError => e
+    #   puts "Syntax error in DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at line: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # rescue StandardError => e
+    #   puts "Error processing DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # end
+
+    output_path = File.join(base_path, output_file)
+    File.write(output_path, JSON.pretty_generate(to_hash))
+    true # Indicate that processing succeeded
+  end
+
+  # Convert to hash or JSON as required
+  def to_hash
+    @data
+  end
+
+  def to_json(*_args)
+    @data.to_json
+  end
+
+  # Method to send data to an endpoint
+  def send_to_endpoint
+    root_key = @data.keys.first
+    action_type = root_key.to_s
+
+    uri = URI.parse("http://localhost:4567/dsl/#{action_type}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Post.new(uri.path, { 'Content-Type' => 'application/json' })
+    payload = { action_type: action_type, data: @data }
+    request.body = payload.to_json
+
+    response = http.request(request)
+    puts "Response: #{response.code} - #{response.message}"
+    puts "Endpoint: #{uri}"
+  end
+end

data/lib/dsl_process_data.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class DSLProcessData
+  PROCESSORS = [{ file_collector: ProcessFileCollector }].freeze
+
+  # Method to process the JSON file after initial evaluation
+  def process(base_path, input_file, output_file)
+    json_file_path = File.join(base_path, input_file)
+    data = JSON.parse(File.read(json_file_path))
+
+    # Loop through the processors and execute matching ones
+    PROCESSORS.each do |processor_entry|
+      key, processor_class = processor_entry.first
+      processor = processor_class.new(key)
+
+      next unless processor.match?(data)
+
+      result = processor.execute(data)
+
+      data['process-data'] ||= {}
+
+      result.each do |key, result|
+        data['process-data'][key.to_s] = result unless result.empty?
+      end
+    end
+
+    # Write the updated JSON data to an extended file
+    extended_output_file = File.join(base_path, output_file)
+    File.write(extended_output_file, JSON.pretty_generate(data))
+  end
+end

data/lib/klue/langcraft/dsl/interpreter.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/6719840b-c72c-8002-bbc2-bbd95fd98d31
+
+module Klue
+  module Langcraft
+    module DSL
+      # Interpreter class for processing and interpreting DSL input
+      #
+      # This class is responsible for handling method calls, processing arguments,
+      # and managing the overall interpretation of the DSL input. It also provides
+      # methods for processing input and output, as well as converting the data
+      # to hash and JSON formats.
+      class Interpreter
+        attr_reader :data
+        attr_accessor :processed
+
+        def initialize
+          klue_reset
+        end
+
+        def process(input: nil, input_file: nil, output_file: nil)
+          klue_reset
+          klue_validate_input_arguments(input, input_file)
+          klue_input_content = klue_input_content(input, input_file)
+
+          @processed = true
+          instance_eval(klue_input_content)
+
+          klue_write_output(output_file) if output_file
+          data
+        end
+
+        def method_missing(method_name, *args, &block)
+          raise "You must call 'process' before using other methods" unless @processed
+
+          key = method_name
+          value = klue_process_args(args, block)
+
+          if @data[key]
+            @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+            @data[key] << value
+          else
+            @data[key] = value
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @processed || super
+        end
+
+        def klue_process_args(args, block)
+          positional_args = []
+          named_args = {}
+
+          # Handling positional and named parameters separately
+          args.each do |arg|
+            if arg.is_a?(Hash)
+              named_args.merge!(arg)
+            else
+              positional_args << arg
+            end
+          end
+
+          # Assign positional parameters generically
+          data = positional_args.each_with_index.to_h { |arg, index| [:"p#{index + 1}", arg] }
+
+          # Merge named parameters after positional ones
+          data.merge!(named_args)
+
+          # Handling a nested block
+          if block
+            interpreter = Interpreter.new
+            interpreter.instance_variable_set(:@processed, true) # Set @processed to true for nested interpreter
+            interpreter.instance_eval(&block)
+            data.merge!(interpreter.data)
+          end
+
+          data.empty? ? nil : data
+        end
+
+        private
+
+        def klue_reset
+          @data = {}
+          @processed = false
+        end
+
+        def klue_validate_input_arguments(input, input_file)
+          raise ArgumentError, 'Either input or input_file must be provided' unless input || input_file
+          raise ArgumentError, 'Both input and input_file cannot be provided' if input && input_file
+        end
+
+        def klue_input_content(input, input_file)
+          input_file ? File.read(input_file) : input
+        end
+
+        def klue_write_output(output_file)
+          output_path = klue_output_path(output_file)
+          File.write(output_path, JSON.pretty_generate(data))
+        end
+
+        def klue_output_path(output_file)
+          if Pathname.new(output_file).absolute?
+            output_file
+          else
+            File.join(File.dirname(output_file), File.basename(output_file))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/klue_runner.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # KlueRunner handles the processing of DSL input data
+      # It manages the execution of various processors and the output of processed data
+      class KlueRunner
+        attr_reader :interpreter, :pipeline, :webhook
+
+        def initialize
+          @interpreter = Klue::Langcraft::DSL::Interpreter.new
+          @pipeline = Klue::Langcraft::DSL::ProcessDataPipeline.new(Klue::Langcraft::DSL::ProcessMatcher.new)
+          @webhook = Klue::Langcraft::DSL::Webhook.new
+        end
+
+        # Run the KlueRunner with the given input data
+        # @param input [String] The input data to process
+        # @param input_file [String] The input file to process (input data and file are mutually exclusive)
+        # @param basic_output_file [String] The output file to write the processed data, this file is before any processing
+        # @param enhanced_output_file [String] The output file to write the processed data, this file is after processing
+        def run(
+          input: nil,
+          input_file: nil,
+          basic_output_file: nil,
+          enhanced_output_file: nil,
+          webhook_url: nil,
+          log_level: :none
+        )
+          @log_level = log_level
+
+          log_info('Processing input')
+          data = interpreter.process(input: input, input_file: input_file, output_file: basic_output_file)
+          log_detailed('Interpreter output:', data)
+
+          log_info('Executing pipeline - enhance')
+          enhanced_data = pipeline.execute(data)
+          log_detailed('Enhanced output:', enhanced_data)
+
+          if enhanced_output_file
+            log_info("Writing enhanced output to file: #{enhanced_output_file}")
+            @pipeline.write_output(enhanced_data, enhanced_output_file)
+          end
+
+          if webhook_url
+            log_info("Delivering data to webhook: #{webhook_url}")
+            @webhook.deliver(webhook_url, enhanced_data)
+          end
+
+          log_info('Processing complete')
+        end
+
+        private
+
+        def log_info(message)
+          puts message if %i[info detailed].include?(@log_level)
+        end
+
+        def log_detailed(message, data)
+          return unless @log_level == :detailed
+
+          puts message
+          pp data
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_data_pipeline.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessDataPipeline class for executing data processing pipelines
+      #
+      # This class is responsible for executing a series of data processing steps
+      # based on the configured processors. It manages the flow of data through
+      # the pipeline and handles the storage of results.
+      class ProcessDataPipeline
+        def initialize(matcher)
+          @matcher = matcher # Use the matcher to find processors
+        end
+
+        # Execute the pipeline of processors on the input data
+        def execute(data)
+          # NOTE: This is the complete data object, each processor will get a cloned version the specific data it is matched to
+          matched_processors = @matcher.match_processors(data)
+
+          matched_processors.each do |processor|
+            processed_data = processor.build_result
+
+            # Store the processed data into the result structure
+            store_result(data, processor, processed_data)
+          end
+
+          data
+        end
+
+        # Optionally write the output to a file
+        def write_output(data, output_file)
+          File.write(output_file, JSON.pretty_generate(data))
+        end
+
+        private
+
+        # Store the processed result back into the data structure
+        def store_result(data, _processor, processed_data)
+          return unless processed_data
+
+          data['process-data'] ||= {}
+
+          if processed_data[:name].nil? || processed_data[:name].empty?
+            index = calculate_index(data, processed_data[:type])
+            processed_data[:name] = "#{processed_data[:type]}-#{index}"
+          end
+
+          data['process-data'][processed_data[:name]] = processed_data
+        end
+
+        def calculate_index(data, processor_type)
+          # Find all keys in 'process-data' that match the processor type (e.g., file_collector)
+          last_index = data['process-data'].keys
+                                           .select { |k| k.start_with?(processor_type.to_s) }  # Keys that start with processor type
+                                           .map { |k| k.split('-').last.to_i }                 # Extract numeric suffix
+                                           .max
+
+          # If no entries exist, start at 1; otherwise, increment the last index
+          last_index ? last_index + 1 : 1
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_matcher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessMatcher class for matching processors to input nodes
+      #
+      # This class is responsible for traversing input nodes and finding
+      # the appropriate processor for each node based on the configured
+      # processor rules.
+      class ProcessMatcher
+        def initialize(processor_config = ProcessorConfigDefault)
+          @processor_config = processor_config
+        end
+
+        def match_processors(nodes)
+          matched_processors = []
+
+          traverse_nodes(nodes) do |key, value|
+            processor_class = find_processor_for(key, value)
+            if processor_class
+              if value.is_a?(Array)
+                # If the value is an array, instantiate a processor for each element
+                value.each do |element|
+                  matched_processors << processor_class.new(element, key)
+                end
+              else
+                matched_processors << processor_class.new(value, key)
+              end
+            end
+          end
+
+          matched_processors
+        end
+
+        private
+
+        def traverse_nodes(node, &block)
+          if node.is_a?(Hash)
+            node.each do |key, value|
+              yield(key, value)
+              traverse_nodes(value, &block)
+            end
+          elsif node.is_a?(Array)
+            node.each_with_index do |item, index|
+              # Provide the index to uniquely identify each element
+              traverse_nodes(item) { |key, value| yield("#{key}[#{index}]", value) }
+            end
+          end
+        end
+
+        # Find the correct processor based on the key using the registered processor config
+        def find_processor_for(key, _value)
+          @processor_config.processor_for(key) # Return the processor class, not an instance
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processor_config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessorConfig class for managing processor configurations
+      #
+      # This class is responsible for registering processors and providing
+      # methods to retrieve processors based on keys or to get all registered
+      # processors.
+      class ProcessorConfig
+        def initialize
+          @processors = {}
+        end
+
+        # Register a processor with its associated keys
+        def register_processor(processor_class)
+          keys = processor_class.keys
+          keys = [keys] unless keys.is_a?(Array)
+          keys.each { |key| @processors[key.to_sym] = processor_class }
+        end
+
+        # Find the processor class by key
+        def processor_for(key)
+          @processors[key.to_sym]
+        end
+
+        # List all registered processors
+        def all_processors
+          @processors
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/file_collector_processor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FileCollectorProcessor class for processing file-related data
+        #
+        # This processor is responsible for handling file collection operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for file-related processing.
+        class FileCollectorProcessor < Processor
+          def self.keys
+            [:file_collector]
+          end
+
+          # Example of how a subclass can implement its specific data logic
+          def build_result_data
+            {
+              files: ['file1.txt', 'file2.txt']
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/full_name_processor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FullNameProcessor class for processing full name data
+        #
+        # This processor is responsible for handling full name processing operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for full name-related processing.
+        class FullNameProcessor < Processor
+          def self.keys
+            [:full_name] # FullNameProcessor's specific key(s)
+          end
+
+          # Implementation of logic for building full name data
+          def build_result_data
+            first_name = data['first_name'] || 'John'
+            last_name = data['last_name'] || 'Doe'
+            full_name = "#{first_name} #{last_name}"
+
+            {
+              full_name: full_name
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/processor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # Base Processor class for defining common processor behavior
+        #
+        # This abstract class serves as the foundation for all specific processors.
+        # It defines the basic structure and common methods that all processors
+        # should implement or inherit.
+        class Processor
+          attr_reader :data, :key
+
+          # Every processor subclass must accept data and key
+          def initialize(data, key)
+            @data = Marshal.load(Marshal.dump(data)) # Deep clone of the data
+            @key = key
+          end
+
+          # Build an envelope result with type, name, and data
+          def build_result
+            {
+              name: data.is_a?(Hash) ? data['as'] : nil,
+              type: key.to_s,
+              data: build_result_data
+            }
+          end
+
+          # Subclasses should override this method to build the actual data.
+          def build_result_data
+            raise NotImplementedError, 'Subclasses must implement `build_data` to generate their specific data'
+          end
+
+          # This will be overridden by subclasses to define keys (or aliases)
+          def self.keys
+            raise NotImplementedError, 'Subclasses must define the `keys` method'
+          end
+        end
+      end
+    end
+  end
+end