klue-langcraft 0.1.0 → 0.2.0

data/lib/klue/langcraft/dsl/interpreter.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/6719840b-c72c-8002-bbc2-bbd95fd98d31
+
+module Klue
+  module Langcraft
+    module DSL
+      # Interpreter class for processing and interpreting DSL input
+      #
+      # This class is responsible for handling method calls, processing arguments,
+      # and managing the overall interpretation of the DSL input. It also provides
+      # methods for processing input and output, as well as converting the data
+      # to hash and JSON formats.
+      class Interpreter
+        attr_reader :data
+        attr_accessor :processed
+
+        def initialize
+          @data = {}
+          @processed = false
+        end
+
+        def process(input: nil, input_file: nil, output_file: nil)
+          validate_input_arguments(input, input_file)
+          input_content = input_content(input, input_file)
+
+          @processed = true
+          instance_eval(input_content)
+
+          write_output(output_file) if output_file
+          data
+        end
+
+        def method_missing(method_name, *args, &block)
+          raise "You must call 'process' before using other methods" unless @processed
+
+          key = method_name
+          value = process_args(args, block)
+
+          if @data[key]
+            @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+            @data[key] << value
+          else
+            @data[key] = value
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @processed || super
+        end
+
+        def process_args(args, block)
+          positional_args = []
+          named_args = {}
+
+          # Handling positional and named parameters separately
+          args.each do |arg|
+            if arg.is_a?(Hash)
+              named_args.merge!(arg)
+            else
+              positional_args << arg
+            end
+          end
+
+          # Assign positional parameters generically
+          data = positional_args.each_with_index.to_h { |arg, index| [:"p#{index + 1}", arg] }
+
+          # Merge named parameters after positional ones
+          data.merge!(named_args)
+
+          # Handling a nested block
+          if block
+            interpreter = Interpreter.new
+            interpreter.instance_variable_set(:@processed, true) # Set @processed to true for nested interpreter
+            interpreter.instance_eval(&block)
+            data.merge!(interpreter.data)
+          end
+
+          data.empty? ? nil : data
+        end
+
+        private
+
+        def validate_input_arguments(input, input_file)
+          raise ArgumentError, 'Either input or input_file must be provided' unless input || input_file
+          raise ArgumentError, 'Both input and input_file cannot be provided' if input && input_file
+        end
+
+        def input_content(input, input_file)
+          input_file ? File.read(input_file) : input
+        end
+
+        def write_output(output_file)
+          output_path = get_output_path(output_file)
+          File.write(output_path, JSON.pretty_generate(data))
+        end
+
+        def get_output_path(output_file)
+          if Pathname.new(output_file).absolute?
+            output_file
+          else
+            File.join(File.dirname(output_file), File.basename(output_file))
+          end
+        end
+
+        # Convert to JSON
+        def to_json(*_args)
+          @data.to_json
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_data_pipeline.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessDataPipeline class for executing data processing pipelines
+      #
+      # This class is responsible for executing a series of data processing steps
+      # based on the configured processors. It manages the flow of data through
+      # the pipeline and handles the storage of results.
+      class ProcessDataPipeline
+        def initialize(matcher)
+          @matcher = matcher # Use the matcher to find processors
+        end
+
+        # Execute the pipeline of processors on the input data
+        def execute(data)
+          # NOTE: This is the complete data object, each processor will get a cloned version the specific data it is matched to
+          matched_processors = @matcher.match_processors(data)
+
+          matched_processors.each do |processor|
+            processed_data = processor.build_result
+
+            # Store the processed data into the result structure
+            store_result(data, processor, processed_data)
+          end
+
+          data
+        end
+
+        # Optionally write the output to a file
+        def write_output(data, output_file)
+          File.write(output_file, JSON.pretty_generate(data))
+        end
+
+        private
+
+        # Store the processed result back into the data structure
+        def store_result(data, _processor, processed_data)
+          return unless processed_data
+
+          data['process-data'] ||= {}
+
+          if processed_data[:name].nil? || processed_data[:name].empty?
+            index = calculate_index(data, processed_data[:type])
+            processed_data[:name] = "#{processed_data[:type]}-#{index}"
+          end
+
+          data['process-data'][processed_data[:name]] = processed_data
+        end
+
+        def calculate_index(data, processor_type)
+          # Find all keys in 'process-data' that match the processor type (e.g., file_collector)
+          last_index = data['process-data'].keys
+                                           .select { |k| k.start_with?(processor_type.to_s) }  # Keys that start with processor type
+                                           .map { |k| k.split('-').last.to_i }                 # Extract numeric suffix
+                                           .max
+
+          # If no entries exist, start at 1; otherwise, increment the last index
+          last_index ? last_index + 1 : 1
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_matcher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessMatcher class for matching processors to input nodes
+      #
+      # This class is responsible for traversing input nodes and finding
+      # the appropriate processor for each node based on the configured
+      # processor rules.
+      class ProcessMatcher
+        def initialize(processor_config = ProcessorConfigDefault)
+          @processor_config = processor_config
+        end
+
+        def match_processors(nodes)
+          matched_processors = []
+
+          traverse_nodes(nodes) do |key, value|
+            processor_class = find_processor_for(key, value)
+            if processor_class
+              if value.is_a?(Array)
+                # If the value is an array, instantiate a processor for each element
+                value.each do |element|
+                  matched_processors << processor_class.new(element, key)
+                end
+              else
+                matched_processors << processor_class.new(value, key)
+              end
+            end
+          end
+
+          matched_processors
+        end
+
+        private
+
+        def traverse_nodes(node, &block)
+          if node.is_a?(Hash)
+            node.each do |key, value|
+              yield(key, value)
+              traverse_nodes(value, &block)
+            end
+          elsif node.is_a?(Array)
+            node.each_with_index do |item, index|
+              # Provide the index to uniquely identify each element
+              traverse_nodes(item) { |key, value| yield("#{key}[#{index}]", value) }
+            end
+          end
+        end
+
+        # Find the correct processor based on the key using the registered processor config
+        def find_processor_for(key, _value)
+          @processor_config.processor_for(key) # Return the processor class, not an instance
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processor_config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessorConfig class for managing processor configurations
+      #
+      # This class is responsible for registering processors and providing
+      # methods to retrieve processors based on keys or to get all registered
+      # processors.
+      class ProcessorConfig
+        def initialize
+          @processors = {}
+        end
+
+        # Register a processor with its associated keys
+        def register_processor(processor_class)
+          keys = processor_class.keys
+          keys = [keys] unless keys.is_a?(Array)
+          keys.each { |key| @processors[key.to_sym] = processor_class }
+        end
+
+        # Find the processor class by key
+        def processor_for(key)
+          @processors[key.to_sym]
+        end
+
+        # List all registered processors
+        def all_processors
+          @processors
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/file_collector_processor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FileCollectorProcessor class for processing file-related data
+        #
+        # This processor is responsible for handling file collection operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for file-related processing.
+        class FileCollectorProcessor < Processor
+          def self.keys
+            [:file_collector]
+          end
+
+          # Example of how a subclass can implement its specific data logic
+          def build_result_data
+            {
+              files: ['file1.txt', 'file2.txt']
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/full_name_processor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FullNameProcessor class for processing full name data
+        #
+        # This processor is responsible for handling full name processing operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for full name-related processing.
+        class FullNameProcessor < Processor
+          def self.keys
+            [:full_name] # FullNameProcessor's specific key(s)
+          end
+
+          # Implementation of logic for building full name data
+          def build_result_data
+            first_name = data['first_name'] || 'John'
+            last_name = data['last_name'] || 'Doe'
+            full_name = "#{first_name} #{last_name}"
+
+            {
+              full_name: full_name
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/processor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # Base Processor class for defining common processor behavior
+        #
+        # This abstract class serves as the foundation for all specific processors.
+        # It defines the basic structure and common methods that all processors
+        # should implement or inherit.
+        class Processor
+          attr_reader :data, :key
+
+          # Every processor subclass must accept data and key
+          def initialize(data, key)
+            @data = Marshal.load(Marshal.dump(data)) # Deep clone of the data
+            @key = key
+          end
+
+          # Build an envelope result with type, name, and data
+          def build_result
+            {
+              name: data.is_a?(Hash) ? data['as'] : nil,
+              type: key.to_s,
+              data: build_result_data
+            }
+          end
+
+          # Subclasses should override this method to build the actual data.
+          def build_result_data
+            raise NotImplementedError, 'Subclasses must implement `build_data` to generate their specific data'
+          end
+
+          # This will be overridden by subclasses to define keys (or aliases)
+          def self.keys
+            raise NotImplementedError, 'Subclasses must define the `keys` method'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/{-brief.md → tokenizer-old-needs-revisit/-brief.md}

@@ -1,5 +1,7 @@
 # Brief for creating the Parser, Tokenizer and Parsing our first DSL
 
+[ChatGPT Conversation](https://chatgpt.com/share/66efd141-6644-8002-970d-4ad641c54d00)
+
 # 1. Parsing Libraries in Ruby
 
 Here are three Ruby parsing libraries with their pros and cons:

data/lib/klue/langcraft/version.rb

@@ -2,6 +2,6 @@
 
 module Klue
   module Langcraft
-    VERSION = '0.1.0'
+    VERSION = '0.2.0'
   end
 end

data/lib/klue/langcraft.rb

@@ -1,6 +1,30 @@
 # frozen_string_literal: true
 
+require 'json'
+require 'net/http'
+require 'uri'
+require 'listen'
+require 'pry'
+
+require 'appydave/tools'
+
 require 'klue/langcraft/version'
+require 'klue/langcraft/dsl/processor_config'
+
+ProcessorConfigDefault = Klue::Langcraft::DSL::ProcessorConfig.new
+
+require 'klue/langcraft/dsl/processors/processor'
+require 'klue/langcraft/dsl/processors/file_collector_processor'
+require 'klue/langcraft/dsl/processors/full_name_processor'
+require 'klue/langcraft/dsl/interpreter'
+require 'klue/langcraft/dsl/process_matcher'
+require 'klue/langcraft/dsl/process_data_pipeline'
+
+require 'base_process'
+require 'process_file_collector'
+require 'dsl_interpreter'
+require 'dsl_folder_watcher'
+require 'dsl_process_data'
 
 module Klue
   module Langcraft
@@ -12,8 +36,8 @@ module Klue
 end
 
 if ENV.fetch('KLUE_DEBUG', 'false').downcase == 'true'
-  namespace = 'KlueLangcraft::Version'
+  namespace = 'Klue::Langcraft::Version'
   file_path = $LOADED_FEATURES.find { |f| f.include?('klue-langcraft/version') }
-  version   = KlueLangcraft::VERSION.ljust(9)
+  version   = Klue::Langcraft::VERSION.ljust(9)
   puts "#{namespace.ljust(35)} : #{version.ljust(9)} : #{file_path}"
 end

data/lib/process_file_collector.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# file: lib/process_file_collector.rb
+
+class ProcessFileCollector < BaseProcess
+  def initialize(key)
+    super
+    @found = nil
+    # @matcher = ->(key, value) { key.to_s == 'file_collector' && value.is_a?(Hash) }
+    @matcher = lambda do |key, value|
+      if key.to_s == 'file_collector'
+        if value.is_a?(Array)
+          value.any? { |v| v.is_a?(Hash) }
+        else
+          value.is_a?(Hash)
+        end
+      else
+        false
+      end
+    end
+  end
+
+  def match?(input)
+    @found = deep_match(input, @matcher)
+    !@found.empty?
+  end
+
+  def execute(_input)
+    # Iterate over each `file_collector` found and process individually
+    results = {}
+
+    @found.each do |data|
+      next unless data.is_a?(Hash)
+
+      # Extract the `as` key if present
+      as_key = data['as']
+
+      working_directory = File.expand_path(data['root'])
+
+      options = Appydave::Tools::GptContext::Options.new(
+        working_directory: working_directory,
+        include_patterns: extract_patterns(data.dig('files', 'include')),
+        exclude_patterns: extract_patterns(data.dig('files', 'exclude')),
+        format: 'json',
+        line_limit: data['line_length']
+      )
+
+      collector = Appydave::Tools::GptContext::FileCollector.new(options)
+      json = collector.build
+
+      # Structuring the result under `process-data` with `as` as key
+      result_data = {
+        type: 'file_collector',
+        data: {
+          working_directory: working_directory,
+          files: JSON.parse(json)
+        }
+      }
+
+      # If `as` key exists, use it to store under process-data with that identifier
+      if as_key
+        results[as_key] = result_data
+      else
+        # Generate a unique key if no `as` key is defined
+        unique_key = "file_collector_#{results.size + 1}"
+        results[unique_key] = result_data
+      end
+    end
+
+    results
+  rescue SyntaxError, NameError, NoMethodError => e
+    puts "Ruby evaluation error in ProcessFileCollector: #{e.message}"
+    puts "Error occurred at: #{e.backtrace.first}"
+    {}
+  rescue StandardError => e
+    puts "Unexpected error in ProcessFileCollector: #{e.message}"
+    puts e.backtrace.join("\n")
+    {}
+  end
+
+  private
+
+  def extract_patterns(files_data)
+    if files_data.is_a?(Hash)
+      [files_data['param1']]
+    elsif files_data.is_a?(Array)
+      files_data.map { |entry| entry['param1'] }
+    else
+      []
+    end
+  end
+end

data/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "klue-langcraft",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "klue-langcraft",
-      "version": "0.1.0",
+      "version": "0.2.0",
       "devDependencies": {
         "@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
         "@semantic-release/changelog": "^6.0.3",

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "klue-langcraft",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Domain Specific Language Crafting",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: klue-langcraft
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - David Cruwys
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-22 00:00:00.000000000 Z
+date: 2024-10-25 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: appydave-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: k_log
   requirement: !ruby/object:Gem::Requirement
@@ -47,27 +61,41 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/dsl_watcher.rb
 - bin/setup
+- docs/dsl-class-diagram.md
 - docs/dsl-examples.md
 - docs/dsl-rules.md
-- docs/dsl-samples/index.md
 - docs/dsl-samples/youtube-launch-optimizer-old.klue
 - docs/dsl-samples/youtube-launch-optimizer-strawberry.json
 - docs/dsl-samples/youtube-launch-optimizer-strawberry.klue
 - docs/dsl-samples/youtube-launch-optimizer.defn.klue
 - docs/dsl-samples/youtube-launch-optimizer.json
 - docs/dsl-samples/youtube-launch-optimizer.klue
+- docs/dsl-upgrade-plan.md
 - docs/project-plan/project-plan.md
 - docs/project-plan/project.drawio
 - docs/project-plan/project_done.svg
 - docs/project-plan/project_in_progress.svg
 - docs/project-plan/project_todo.svg
+- lib/base_process.rb
+- lib/dsl_folder_watcher.rb
+- lib/dsl_interpreter.rb
+- lib/dsl_process_data.rb
 - lib/klue/langcraft.rb
-- lib/klue/langcraft/-brief.md
-- lib/klue/langcraft/parser.rb
-- lib/klue/langcraft/sample_usage.rb
-- lib/klue/langcraft/tokenizer.rb
+- lib/klue/langcraft/dsl/interpreter.rb
+- lib/klue/langcraft/dsl/process_data_pipeline.rb
+- lib/klue/langcraft/dsl/process_matcher.rb
+- lib/klue/langcraft/dsl/processor_config.rb
+- lib/klue/langcraft/dsl/processors/file_collector_processor.rb
+- lib/klue/langcraft/dsl/processors/full_name_processor.rb
+- lib/klue/langcraft/dsl/processors/processor.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/-brief.md
+- lib/klue/langcraft/tokenizer-old-needs-revisit/parser.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/sample_usage.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/tokenizer.rb
 - lib/klue/langcraft/version.rb
+- lib/process_file_collector.rb
 - package-lock.json
 - package.json
 - sig/klue/langcraft.rbs

data/docs/dsl-samples/index.md

@@ -1,4 +0,0 @@
-Printspeak DSL
-Project Plan
-Agent As Code
-Make Chapters out of my recordings using a folder and glob pattern geared to my naming convention