klue-langcraft 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c27050239df45678f9bf8b8527c20f033532235d023cf22bd255b06f0f48b38
-  data.tar.gz: 22e0fab109f694eb0fc501885f38e94b8e0eae55da1e66381093fc8cedbece86
+  metadata.gz: '07528e25c004b36664752d235737a9f6f521e231eb2b9e900884861e2fbeccad'
+  data.tar.gz: bd96ac7b1247f8f4fe123b4bb7923df50f1067120b1faa6e1121c6936fe400d1
 SHA512:
-  metadata.gz: 78d038c2e19671fc645c02215820345d9a89f8552869f9eb2f0d0f6769e348bef56f218ff39114bd6f8945ef44001d928b1f5a0f4896e8510ffcebe6ad9f49a3
-  data.tar.gz: 80e94051e6ad5104cba4b199b472bbea3f62bdc23903a71974fb04d9399a92beb23627581ef30c9e02a0f8a821d1a9ef48cc9cb7a19a1cd223a1af95085b9c1c
+  metadata.gz: f270af8583b0ef01f5c290f0d69abe2de5534682169440381532fea9510c0b2b03f7c247c13dcde3fd13dc88b89fdadc3255af804138b6373513ea16a32b31d6
+  data.tar.gz: e23279553bc9f644ebe56ce796dd99cda278c10170142806e2a9ed59839a3a51a541de9b0b75082474e46f130cd6e3e5d037d2c5e9c475585280ba032d84a6d8

data/.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
   Exclude:
     - ".builders/**/*"
     - "spec/samples/**/*"
+    - "lib/*" # OLD DSL INTERPRETER
 
 Metrics/BlockLength:
   Exclude:
@@ -104,3 +105,14 @@ RSpec/SpecFilePathSuffix:
 RSpec/NamedSubject:
   Exclude:
     - "**/spec/**/*"
+
+RSpec/MultipleExpectations:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/MultipleMemoizedHelpers:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/ExampleLength:
+  Max: 20

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [0.1.1](https://github.com/appydave/klue-langcraft/compare/v0.1.0...v0.1.1) (2024-09-22)
+
+
+### Bug Fixes
+
+* add chatgpt conversation link ([ae5448d](https://github.com/appydave/klue-langcraft/commit/ae5448d4e23ab1fd673c788e6f30e736d82afb44))
+
 # [0.1.0](https://github.com/appydave/klue-langcraft/compare/v0.0.7...v0.1.0) (2024-09-22)
 
 

data/bin/dsl_watcher.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+
+require 'klue/langcraft'
+
+BASE_PATH = ARGV[0] || '/Users/davidcruwys/dev/appydave/klueless'
+
+DSLFolderWatcher.watch(BASE_PATH)

data/docs/dsl-class-diagram.md

@@ -0,0 +1,97 @@
+
+# DSL Class Diagram
+
+```plaintext
++----------------------------+          
+|      Klue::Langcraft::DSL   |
+|----------------------------|
+| + Interpreter               |
+| + ProcessMatcher            |
+| + ProcessorConfig           |
+| + ProcessDataPipeline       |
+| + Processors                |
++----------------------------+
+
+                |
+                v
++-----------------------------------+
+|          Interpreter              |         
+|-----------------------------------|           
+| - data: Hash                      |           
+| - processed: Boolean              |
+|-----------------------------------|           
+| + initialize()                    |
+| + process(input, output)          |
+| + method_missing()                |
+| + process_args(args, block)       |
+| + respond_to_missing?()           |
+| + to_json()                       |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|         ProcessMatcher            |         
+|-----------------------------------|           
+| + match_processors(nodes)         |           
+|-----------------------------------|
+| - traverse_nodes(node, &block)    |
+| - find_processor_for(key, value)  |
+|-----------------------------------|
+| - processor_config: ProcessorConfig|
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|      ProcessDataPipeline          |         
+|-----------------------------------|           
+| + execute(data)                   |           
+| + write_output(data, output_file) |
+|-----------------------------------|
+| - store_result(data, processor,   |
+|   processed_data)                 |
+| - calculate_index(data, processor)|
+|-----------------------------------|
+| - matcher: ProcessMatcher         |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|       ProcessorConfig             |         
+|-----------------------------------|           
+| + register_processor(processor)   |           
+| + processor_for(key)              |
+| + all_processors()                |
+|-----------------------------------|
+| - processors: Hash                |
++-----------------------------------+
+
+                |
+                v
++----------------------------+          
+|         Processor           |          
+|----------------------------|           
+| - data: Hash                |
+| - key: Symbol               |
+|----------------------------|
+| + initialize(data, key)     |
+| + build_result()            |
+| + build_result_data()       |
+|----------------------------|
+| + keys() (abstract method)  |
++----------------------------+
+
+                |
+                v
++----------------------------+           +-------------------------+
+|  FileCollectorProcessor     |           |     FullNameProcessor   |
+|----------------------------|           +-------------------------+
+| (inherits Processor)        |           | (inherits Processor)    |
+|----------------------------|           +-------------------------+
+| + build_result_data()       |           | + build_result_data()   |
+| + Auto-register with        |           | + Auto-register with    |
+|   ProcessorConfig           |           |   ProcessorConfig       |
++----------------------------+           +-------------------------+
+```

data/docs/dsl-examples.md

@@ -103,3 +103,12 @@ definition :workflow do
   end
 end
 ```
+
+
+You're going to help me process a Ruby inspired DSL into JSON.
+
+There will be two parts of this conversation, part one will be about exploring the concept with a simple reflection based system that will work with any Ruby compatible structure.
+
+Part two, which I kick off after I finish part one will be to use an AST style of approach so that I can process the same concept, JavaScript or RUST.
+
+To get started, I'm going to give you some sample Ruby DSL and you're going to convert them to an equivalent ruby hash

data/docs/dsl-upgrade-plan.md

@@ -0,0 +1,266 @@
+# DSL System Expansion Plan
+
+## Overview
+
+The current system processes DSL input using an interpreter, matchers, processors, and pipelines. We need to expand the system by introducing several new features:
+
+1. **Processor Service**: A centralized service that orchestrates all operations, simplifying how the watcher and command-line tools work.
+2. **Data Saving Option**: An option to toggle whether the processed JSON data is saved to a file or not.
+3. **Endpoint for Data Delivery**: Add the capability to send the processed data to external HTTP endpoints, configurable via the command line.
+
+The changes will affect the class structure, as well as introduce new command-line options and configurations. Below is a detailed explanation of the required features and how they will be implemented.
+
+---
+
+## New Components and Changes
+
+### 1. **Processor Service**
+
+- **Purpose**: We need to introduce a new service (tentatively called `ProcessorService`) to orchestrate the different components. This class will:
+  - Call the interpreter to process the DSL input.
+  - Run the data through the processing pipeline to apply any necessary transformations.
+  - Handle the saving of the processed data to a file if configured.
+  - Send the resulting JSON data to external endpoints if URLs are provided via the command line.
+
+- **Why**: Currently, the watcher or command-line tool is responsible for calling various components (interpreter, pipeline, etc.). By introducing this service, we centralize the logic and simplify the external interfaces. The watcher will only need to call this service, and any configuration options (like file saving or sending data to an endpoint) will be handled internally by the service.
+
+- **Key Features**:
+  - **Options Handling**: The service will take in various options, such as:
+    - Whether to save the processed JSON to a file.
+    - URLs for sending data (e.g., one for simplified JSON, one for enhanced JSON).
+  - **Sequential Processing**: The service will sequentially call:
+    1. The interpreter to process the DSL.
+    2. The processing pipeline to enhance the data.
+    3. Save the data to a file (if the option is enabled).
+    4. Send the data to the endpoint (if URLs are provided).
+
+### 2. **Data Saving Option**
+
+- **Purpose**: Introduce a command-line option (`--save-to-file`) that will enable or disable saving the processed data to a file. By default, the data should flow through the system, but whether it gets written to the filesystem should be configurable.
+
+- **Why**: Not all users may want to persist the JSON data to disk. The ability to toggle this behavior will make the system more flexible.
+
+- **Details**:
+  - When the `--save-to-file` option is enabled, the processed data should be saved to a file.
+  - If the option is not provided, the data will not be saved to a file, but it will still flow through the system for other uses (e.g., sending to endpoints).
+  
+### 3. **Data Delivery to Endpoints**
+
+- **Purpose**: Add the ability to send the JSON data (either simplified or enhanced) to external HTTP endpoints. These endpoints should be configurable via command-line options (`--endpoint-simplified` and `--endpoint-enhanced`).
+
+- **Why**: The system should be able to integrate with external services by sending the resulting data to a web application. For example, the web app could receive simplified JSON for debugging and enhanced JSON for final usage. This allows seamless data transfer between the DSL processing system and external applications.
+
+- **Details**:
+  - The `ProcessorService` will check if any URLs are provided via the command line.
+  - If the URLs are provided, it will send the resulting JSON data (using HTTP POST) to those endpoints.
+  - **Two endpoints**:
+    - One for simplified JSON.
+    - One for enhanced JSON.
+  - The system should allow sending data to either or both endpoints, depending on the provided options.
+
+### 4. **Command-Line Interface (Expanded)**
+
+- The existing `CommandLineInterface` class should be expanded to handle the new options:
+  - `--save-to-file`: Toggles whether the JSON is saved to a file.
+  - `--endpoint-simplified`: Specifies the URL for sending simplified JSON.
+  - `--endpoint-enhanced`: Specifies the URL for sending enhanced JSON.
+  
+- These options should be passed to the `ProcessorService`, which will handle the actual behavior (e.g., saving the file, sending data to the endpoint).
+  
+### 5. **Watcher Integration**
+
+- The existing `Watcher` class, which uses `ListenGem` to monitor file changes, will now delegate all processing to the `ProcessorService`. When a file change is detected, the watcher should simply call the service with the necessary options.
+  
+- This means that the watcher doesn't need to know about interpreters, pipelines, or processors—it just needs to hand over the file to the `ProcessorService`, which will handle everything sequentially.
+
+---
+
+## Revised Class Structure
+
+### New Classes:
+
+1. **ProcessorService**:
+   - Central orchestrator for calling the interpreter, processing data, saving the file, and sending data to endpoints.
+   - Will take all options from the command line and watcher.
+
+2. **DataDeliveryService**:
+   - Handles sending JSON data to external HTTP endpoints.
+   - Accepts a URL and sends the provided data using HTTP POST.
+
+### Expanded Classes:
+
+1. **CommandLineInterface**:
+   - Now handles additional command-line options (`--save-to-file`, `--endpoint-simplified`, `--endpoint-enhanced`).
+   - Passes these options to the `ProcessorService`.
+
+2. **Watcher**:
+   - Instead of directly calling interpreters and pipelines, the watcher will now pass the detected file changes to the `ProcessorService` for handling.
+
+---
+
+## Command-Line Options
+
+- **New Options**:
+  - `--save-to-file`: If provided, the resulting JSON will be saved to a file. Otherwise, the data will only flow through the system.
+  - `--endpoint-simplified`: A URL for sending the simplified JSON to an external endpoint.
+  - `--endpoint-enhanced`: A URL for sending the enhanced JSON to an external endpoint.
+
+These options should be passed to the `ProcessorService`, which will then handle the appropriate behavior based on the configuration.
+
+---
+
+## Final Workflow
+
+1. **Command-Line Usage**:
+   - Users can pass options such as directories to watch, whether to save files, and URLs for endpoints.
+   
+2. **Watcher**:
+   - Monitors the specified directories for file changes and passes any detected changes to the `ProcessorService`.
+   
+3. **ProcessorService**:
+   - Orchestrates the sequence:
+     1. Calls the `Interpreter` to process the DSL input.
+     2. Runs the `ProcessDataPipeline` to enhance the data.
+     3. Saves the resulting data to a file if the `--save-to-file` option is enabled.
+     4. Sends the resulting data to the provided URL(s) if the `--endpoint-simplified` or `--endpoint-enhanced` options are specified.
+
+4. **Data Delivery**:
+   - The `DataDeliveryService` is responsible for sending the processed data to the external endpoints, handling any HTTP interactions required.
+
+
+# SAMPLE CODE, NOT TESTED BUT MIGHT BE SUITABLE
+```ruby
+
+# file: lib/klue/langcraft/dsl/data_delivery_service.rb
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Klue
+  module Langcraft
+    module DSL
+      class DataDeliveryService
+        def initialize(url)
+          @url = URI.parse(url)
+        end
+
+        def send(data)
+          http = Net::HTTP.new(@url.host, @url.port)
+          request = Net::HTTP::Post.new(@url.request_uri, { 'Content-Type' => 'application/json' })
+          request.body = data.to_json
+          response = http.request(request)
+
+          puts "Data sent to #{@url}: #{response.code} #{response.message}"
+        end
+      end
+    end
+  end
+end
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/processor_service.rb
+require_relative 'interpreter'
+require_relative 'process_data_pipeline'
+require_relative 'data_delivery_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class ProcessorService
+        def initialize(options)
+          @options = options
+          @interpreter = Interpreter.new
+          @pipeline = ProcessDataPipeline.new(ProcessMatcher.new)
+        end
+
+        def run
+          data = call_interpreter
+          enhanced_data = run_pipeline(data)
+          save_to_file(enhanced_data) if @options[:save_to_file]
+          send_to_endpoint(enhanced_data)
+        end
+
+        private
+
+        def call_interpreter
+          @interpreter.process(input: 'path_to_input_file.dsl')
+        end
+
+        def run_pipeline(data)
+          @pipeline.execute(data)
+        end
+
+        def save_to_file(data)
+          File.write('output.json', JSON.pretty_generate(data))
+        end
+
+        def send_to_endpoint(data)
+          if @options[:endpoint_simplified]
+            DataDeliveryService.new(@options[:endpoint_simplified]).send(data)
+          end
+
+          if @options[:endpoint_enhanced]
+            DataDeliveryService.new(@options[:endpoint_enhanced]).send(data)
+          end
+        end
+      end
+    end
+  end
+end
+
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/command_line_interface.rb
+require_relative 'watcher'
+require_relative 'processor_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class CommandLineInterface
+        def initialize
+          @watch_dirs = []
+          @options = {}
+        end
+
+        def start
+          parse_arguments
+          start_watcher_or_processor_service
+        end
+
+        private
+
+        def parse_arguments
+          ARGV.each_with_index do |arg, index|
+            case arg
+            when '--watch'
+              @watch_dirs << ARGV[index + 1]
+            when '--save-to-file'
+              @options[:save_to_file] = true
+            when '--endpoint-simplified'
+              @options[:endpoint_simplified] = ARGV[index + 1]
+            when '--endpoint-enhanced'
+              @options[:endpoint_enhanced] = ARGV[index + 1]
+            end
+          end
+
+          raise 'No directories specified for watching' if @watch_dirs.empty?
+        end
+
+        def start_watcher_or_processor_service
+          if @watch_dirs.any?
+            watcher = Watcher.new(@watch_dirs)
+            watcher.start
+          else
+            processor_service = ProcessorService.new(@options)
+            processor_service.run
+          end
+        end
+      end
+    end
+  end
+end
+
+```

data/lib/base_process.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# file: lib/base_process.rb
+
+class BaseProcess
+  attr_reader :key
+
+  def initialize(key)
+    @key = key
+  end
+
+  def deep_match(input, predicate)
+    matches = []
+
+    # If the current input is a Hash, iterate over each key-value pair
+    if input.is_a?(Hash)
+
+      input.each do |key, value|
+
+        # If the value matches the predicate, add it to matches
+        if predicate.call(key, value)
+          matches << value
+        end
+
+        # Continue searching deeper within the value
+        matches.concat(deep_match(value, predicate))
+      end
+
+    # If the input is an Array, iterate over each item
+    elsif input.is_a?(Array)
+
+      input.each do |item|
+
+        # Continue searching within each item of the array
+        matches.concat(deep_match(item, predicate))
+      end
+    end
+
+    matches
+  end
+end

data/lib/dsl_folder_watcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class DSLFolderWatcher
+  def self.watch(folder_path)
+    puts "Watching: #{folder_path}"
+    listener = Listen.to(folder_path) do |modified, added, _removed|
+      changes = (modified + added).uniq
+
+      # DEBOUNCE CURRENTLY NOT WORKING
+      # debounce_map = {}
+      # debounce_interval = 1 # seconds
+
+      changes.each do |file_path|
+        next unless File.extname(file_path) == '.klue'
+
+        puts file_path
+
+        # debounce_map[file_path] ||= Time.now
+        # next unless Time.now - debounce_map[file_path] >= debounce_interval
+
+        # debounce_map[file_path] = Time.now
+
+        base_name = file_path.gsub(/\.klue$/, '')
+        input_file = "#{base_name}.klue"
+        output_file = "#{base_name}.json"
+
+        interpreter = DSLInterpreter.new
+        if interpreter.process('', input_file, output_file)
+          # Process the JSON data to add 'process-data' details
+          dsl_processor = DSLProcessData.new
+          dsl_processor.process('', output_file, output_file)
+          # SKIP EXTEND FILE FOR NOW AND REWRITE THE OUTPUTFILE
+          # dsl_processor.process('', output_file, extended_output_file)
+
+          # interpreter.send_to_endpoint
+        else
+          puts 'Skipping further processing due to errors in DSL interpretation.'
+        end
+      end
+
+      # Remove old entries from debounce_map to prevent memory bloat
+      # debounce_map.each_key do |key|
+      #   debounce_map.delete(key) if Time.now - debounce_map[key] > debounce_interval * 2
+      # end
+    end
+    listener.start
+    puts "Wait for changes: #{folder_path}"
+    sleep
+  end
+end

data/lib/dsl_interpreter.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/67064770-d524-8002-8344-3091e895d150
+# - https://chatgpt.com/c/6706289c-9b9c-8002-86e3-f9198c1c608a
+# - https://chatgpt.com/c/670dcd34-5dbc-8002-ad7a-d4df54a6a2e0
+#
+class DSLInterpreter
+  def initialize
+    @data = {}
+  end
+
+  # Capturing top-level DSL methods
+  def method_missing(method_name, *args, &block)
+    key = method_name
+    value = process_args(args, block)
+
+    # Append key-value to the current context of @data
+    if @data[key]
+      @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+      @data[key] << value
+    else
+      @data[key] = value
+    end
+  end
+
+  # A method to handle parameters and nested blocks
+  def process_args(args, block)
+    data = {}
+
+    # Handling positional and named parameters separately
+    positional_args = []
+    named_args = {}
+
+    args.each do |arg|
+      if arg.is_a?(Hash)
+        named_args.merge!(arg)
+      else
+        positional_args << arg
+      end
+    end
+
+    # Assign positional parameters generically
+    positional_args.each_with_index do |arg, index|
+      data[:"param#{index + 1}"] = arg
+    end
+
+    # Merge named parameters directly
+    data.merge!(named_args)
+
+    # Handling a nested block
+    if block
+      interpreter = DSLInterpreter.new
+      interpreter.instance_eval(&block)
+      data.merge!(interpreter.data)
+    end
+
+    data.empty? ? nil : data
+  end
+
+  # To access data after interpreting
+  attr_reader :data
+
+  # Reading file and evaluating as Ruby
+  def process(base_path, input_file, output_file)
+    file_path = File.join(base_path, input_file)
+    content = File.read(file_path)
+
+    # begin
+    instance_eval(content)
+    # rescue SyntaxError => e
+    #   puts "Syntax error in DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at line: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # rescue StandardError => e
+    #   puts "Error processing DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # end
+
+    output_path = File.join(base_path, output_file)
+    File.write(output_path, JSON.pretty_generate(to_hash))
+    true # Indicate that processing succeeded
+  end
+
+  # Convert to hash or JSON as required
+  def to_hash
+    @data
+  end
+
+  def to_json(*_args)
+    @data.to_json
+  end
+
+  # Method to send data to an endpoint
+  def send_to_endpoint
+    root_key = @data.keys.first
+    action_type = root_key.to_s
+
+    uri = URI.parse("http://localhost:4567/dsl/#{action_type}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Post.new(uri.path, { 'Content-Type' => 'application/json' })
+    payload = { action_type: action_type, data: @data }
+    request.body = payload.to_json
+
+    response = http.request(request)
+    puts "Response: #{response.code} - #{response.message}"
+    puts "Endpoint: #{uri}"
+  end
+end

data/lib/dsl_process_data.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class DSLProcessData
+  PROCESSORS = [{ file_collector: ProcessFileCollector }].freeze
+
+  # Method to process the JSON file after initial evaluation
+  def process(base_path, input_file, output_file)
+    json_file_path = File.join(base_path, input_file)
+    data = JSON.parse(File.read(json_file_path))
+
+    # Loop through the processors and execute matching ones
+    PROCESSORS.each do |processor_entry|
+      key, processor_class = processor_entry.first
+      processor = processor_class.new(key)
+
+      next unless processor.match?(data)
+
+      result = processor.execute(data)
+
+      data['process-data'] ||= {}
+
+      result.each do |key, result|
+        data['process-data'][key.to_s] = result unless result.empty?
+      end
+    end
+
+    # Write the updated JSON data to an extended file
+    extended_output_file = File.join(base_path, output_file)
+    File.write(extended_output_file, JSON.pretty_generate(data))
+  end
+end

data/lib/klue/langcraft/dsl/interpreter.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/6719840b-c72c-8002-bbc2-bbd95fd98d31
+
+module Klue
+  module Langcraft
+    module DSL
+      # Interpreter class for processing and interpreting DSL input
+      #
+      # This class is responsible for handling method calls, processing arguments,
+      # and managing the overall interpretation of the DSL input. It also provides
+      # methods for processing input and output, as well as converting the data
+      # to hash and JSON formats.
+      class Interpreter
+        attr_reader :data
+        attr_accessor :processed
+
+        def initialize
+          @data = {}
+          @processed = false
+        end
+
+        def process(input: nil, input_file: nil, output_file: nil)
+          validate_input_arguments(input, input_file)
+          input_content = input_content(input, input_file)
+
+          @processed = true
+          instance_eval(input_content)
+
+          write_output(output_file) if output_file
+          data
+        end
+
+        def method_missing(method_name, *args, &block)
+          raise "You must call 'process' before using other methods" unless @processed
+
+          key = method_name
+          value = process_args(args, block)
+
+          if @data[key]
+            @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+            @data[key] << value
+          else
+            @data[key] = value
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @processed || super
+        end
+
+        def process_args(args, block)
+          positional_args = []
+          named_args = {}
+
+          # Handling positional and named parameters separately
+          args.each do |arg|
+            if arg.is_a?(Hash)
+              named_args.merge!(arg)
+            else
+              positional_args << arg
+            end
+          end
+
+          # Assign positional parameters generically
+          data = positional_args.each_with_index.to_h { |arg, index| [:"p#{index + 1}", arg] }
+
+          # Merge named parameters after positional ones
+          data.merge!(named_args)
+
+          # Handling a nested block
+          if block
+            interpreter = Interpreter.new
+            interpreter.instance_variable_set(:@processed, true) # Set @processed to true for nested interpreter
+            interpreter.instance_eval(&block)
+            data.merge!(interpreter.data)
+          end
+
+          data.empty? ? nil : data
+        end
+
+        private
+
+        def validate_input_arguments(input, input_file)
+          raise ArgumentError, 'Either input or input_file must be provided' unless input || input_file
+          raise ArgumentError, 'Both input and input_file cannot be provided' if input && input_file
+        end
+
+        def input_content(input, input_file)
+          input_file ? File.read(input_file) : input
+        end
+
+        def write_output(output_file)
+          output_path = get_output_path(output_file)
+          File.write(output_path, JSON.pretty_generate(data))
+        end
+
+        def get_output_path(output_file)
+          if Pathname.new(output_file).absolute?
+            output_file
+          else
+            File.join(File.dirname(output_file), File.basename(output_file))
+          end
+        end
+
+        # Convert to JSON
+        def to_json(*_args)
+          @data.to_json
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_data_pipeline.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessDataPipeline class for executing data processing pipelines
+      #
+      # This class is responsible for executing a series of data processing steps
+      # based on the configured processors. It manages the flow of data through
+      # the pipeline and handles the storage of results.
+      class ProcessDataPipeline
+        def initialize(matcher)
+          @matcher = matcher # Use the matcher to find processors
+        end
+
+        # Execute the pipeline of processors on the input data
+        def execute(data)
+          # NOTE: This is the complete data object, each processor will get a cloned version the specific data it is matched to
+          matched_processors = @matcher.match_processors(data)
+
+          matched_processors.each do |processor|
+            processed_data = processor.build_result
+
+            # Store the processed data into the result structure
+            store_result(data, processor, processed_data)
+          end
+
+          data
+        end
+
+        # Optionally write the output to a file
+        def write_output(data, output_file)
+          File.write(output_file, JSON.pretty_generate(data))
+        end
+
+        private
+
+        # Store the processed result back into the data structure
+        def store_result(data, _processor, processed_data)
+          return unless processed_data
+
+          data['process-data'] ||= {}
+
+          if processed_data[:name].nil? || processed_data[:name].empty?
+            index = calculate_index(data, processed_data[:type])
+            processed_data[:name] = "#{processed_data[:type]}-#{index}"
+          end
+
+          data['process-data'][processed_data[:name]] = processed_data
+        end
+
+        def calculate_index(data, processor_type)
+          # Find all keys in 'process-data' that match the processor type (e.g., file_collector)
+          last_index = data['process-data'].keys
+                                           .select { |k| k.start_with?(processor_type.to_s) }  # Keys that start with processor type
+                                           .map { |k| k.split('-').last.to_i }                 # Extract numeric suffix
+                                           .max
+
+          # If no entries exist, start at 1; otherwise, increment the last index
+          last_index ? last_index + 1 : 1
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/process_matcher.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessMatcher class for matching processors to input nodes
+      #
+      # This class is responsible for traversing input nodes and finding
+      # the appropriate processor for each node based on the configured
+      # processor rules.
+      class ProcessMatcher
+        def initialize(processor_config = ProcessorConfigDefault)
+          @processor_config = processor_config
+        end
+
+        def match_processors(nodes)
+          matched_processors = []
+
+          traverse_nodes(nodes) do |key, value|
+            processor_class = find_processor_for(key, value)
+            if processor_class
+              if value.is_a?(Array)
+                # If the value is an array, instantiate a processor for each element
+                value.each do |element|
+                  matched_processors << processor_class.new(element, key)
+                end
+              else
+                matched_processors << processor_class.new(value, key)
+              end
+            end
+          end
+
+          matched_processors
+        end
+
+        private
+
+        def traverse_nodes(node, &block)
+          if node.is_a?(Hash)
+            node.each do |key, value|
+              yield(key, value)
+              traverse_nodes(value, &block)
+            end
+          elsif node.is_a?(Array)
+            node.each_with_index do |item, index|
+              # Provide the index to uniquely identify each element
+              traverse_nodes(item) { |key, value| yield("#{key}[#{index}]", value) }
+            end
+          end
+        end
+
+        # Find the correct processor based on the key using the registered processor config
+        def find_processor_for(key, _value)
+          @processor_config.processor_for(key) # Return the processor class, not an instance
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processor_config.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      # ProcessorConfig class for managing processor configurations
+      #
+      # This class is responsible for registering processors and providing
+      # methods to retrieve processors based on keys or to get all registered
+      # processors.
+      class ProcessorConfig
+        def initialize
+          @processors = {}
+        end
+
+        # Register a processor with its associated keys
+        def register_processor(processor_class)
+          keys = processor_class.keys
+          keys = [keys] unless keys.is_a?(Array)
+          keys.each { |key| @processors[key.to_sym] = processor_class }
+        end
+
+        # Find the processor class by key
+        def processor_for(key)
+          @processors[key.to_sym]
+        end
+
+        # List all registered processors
+        def all_processors
+          @processors
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/file_collector_processor.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FileCollectorProcessor class for processing file-related data
+        #
+        # This processor is responsible for handling file collection operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for file-related processing.
+        class FileCollectorProcessor < Processor
+          def self.keys
+            [:file_collector]
+          end
+
+          # Example of how a subclass can implement its specific data logic
+          def build_result_data
+            {
+              files: ['file1.txt', 'file2.txt']
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/full_name_processor.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # FullNameProcessor class for processing full name data
+        #
+        # This processor is responsible for handling full name processing operations
+        # within the DSL. It inherits from the base Processor class and implements
+        # specific logic for full name-related processing.
+        class FullNameProcessor < Processor
+          def self.keys
+            [:full_name] # FullNameProcessor's specific key(s)
+          end
+
+          # Implementation of logic for building full name data
+          def build_result_data
+            first_name = data['first_name'] || 'John'
+            last_name = data['last_name'] || 'Doe'
+            full_name = "#{first_name} #{last_name}"
+
+            {
+              full_name: full_name
+            }
+          end
+
+          # Auto-register the processor as soon as the class is loaded
+          ProcessorConfigDefault.register_processor(self)
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/dsl/processors/processor.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Klue
+  module Langcraft
+    module DSL
+      module Processors
+        # Base Processor class for defining common processor behavior
+        #
+        # This abstract class serves as the foundation for all specific processors.
+        # It defines the basic structure and common methods that all processors
+        # should implement or inherit.
+        class Processor
+          attr_reader :data, :key
+
+          # Every processor subclass must accept data and key
+          def initialize(data, key)
+            @data = Marshal.load(Marshal.dump(data)) # Deep clone of the data
+            @key = key
+          end
+
+          # Build an envelope result with type, name, and data
+          def build_result
+            {
+              name: data.is_a?(Hash) ? data['as'] : nil,
+              type: key.to_s,
+              data: build_result_data
+            }
+          end
+
+          # Subclasses should override this method to build the actual data.
+          def build_result_data
+            raise NotImplementedError, 'Subclasses must implement `build_data` to generate their specific data'
+          end
+
+          # This will be overridden by subclasses to define keys (or aliases)
+          def self.keys
+            raise NotImplementedError, 'Subclasses must define the `keys` method'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/klue/langcraft/version.rb

@@ -2,6 +2,6 @@
 
 module Klue
   module Langcraft
-    VERSION = '0.1.1'
+    VERSION = '0.2.0'
   end
 end

data/lib/klue/langcraft.rb

@@ -1,6 +1,30 @@
 # frozen_string_literal: true
 
+require 'json'
+require 'net/http'
+require 'uri'
+require 'listen'
+require 'pry'
+
+require 'appydave/tools'
+
 require 'klue/langcraft/version'
+require 'klue/langcraft/dsl/processor_config'
+
+ProcessorConfigDefault = Klue::Langcraft::DSL::ProcessorConfig.new
+
+require 'klue/langcraft/dsl/processors/processor'
+require 'klue/langcraft/dsl/processors/file_collector_processor'
+require 'klue/langcraft/dsl/processors/full_name_processor'
+require 'klue/langcraft/dsl/interpreter'
+require 'klue/langcraft/dsl/process_matcher'
+require 'klue/langcraft/dsl/process_data_pipeline'
+
+require 'base_process'
+require 'process_file_collector'
+require 'dsl_interpreter'
+require 'dsl_folder_watcher'
+require 'dsl_process_data'
 
 module Klue
   module Langcraft
@@ -12,8 +36,8 @@ module Klue
 end
 
 if ENV.fetch('KLUE_DEBUG', 'false').downcase == 'true'
-  namespace = 'KlueLangcraft::Version'
+  namespace = 'Klue::Langcraft::Version'
   file_path = $LOADED_FEATURES.find { |f| f.include?('klue-langcraft/version') }
-  version   = KlueLangcraft::VERSION.ljust(9)
+  version   = Klue::Langcraft::VERSION.ljust(9)
   puts "#{namespace.ljust(35)} : #{version.ljust(9)} : #{file_path}"
 end

data/lib/process_file_collector.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# file: lib/process_file_collector.rb
+
+class ProcessFileCollector < BaseProcess
+  def initialize(key)
+    super
+    @found = nil
+    # @matcher = ->(key, value) { key.to_s == 'file_collector' && value.is_a?(Hash) }
+    @matcher = lambda do |key, value|
+      if key.to_s == 'file_collector'
+        if value.is_a?(Array)
+          value.any? { |v| v.is_a?(Hash) }
+        else
+          value.is_a?(Hash)
+        end
+      else
+        false
+      end
+    end
+  end
+
+  def match?(input)
+    @found = deep_match(input, @matcher)
+    !@found.empty?
+  end
+
+  def execute(_input)
+    # Iterate over each `file_collector` found and process individually
+    results = {}
+
+    @found.each do |data|
+      next unless data.is_a?(Hash)
+
+      # Extract the `as` key if present
+      as_key = data['as']
+
+      working_directory = File.expand_path(data['root'])
+
+      options = Appydave::Tools::GptContext::Options.new(
+        working_directory: working_directory,
+        include_patterns: extract_patterns(data.dig('files', 'include')),
+        exclude_patterns: extract_patterns(data.dig('files', 'exclude')),
+        format: 'json',
+        line_limit: data['line_length']
+      )
+
+      collector = Appydave::Tools::GptContext::FileCollector.new(options)
+      json = collector.build
+
+      # Structuring the result under `process-data` with `as` as key
+      result_data = {
+        type: 'file_collector',
+        data: {
+          working_directory: working_directory,
+          files: JSON.parse(json)
+        }
+      }
+
+      # If `as` key exists, use it to store under process-data with that identifier
+      if as_key
+        results[as_key] = result_data
+      else
+        # Generate a unique key if no `as` key is defined
+        unique_key = "file_collector_#{results.size + 1}"
+        results[unique_key] = result_data
+      end
+    end
+
+    results
+  rescue SyntaxError, NameError, NoMethodError => e
+    puts "Ruby evaluation error in ProcessFileCollector: #{e.message}"
+    puts "Error occurred at: #{e.backtrace.first}"
+    {}
+  rescue StandardError => e
+    puts "Unexpected error in ProcessFileCollector: #{e.message}"
+    puts e.backtrace.join("\n")
+    {}
+  end
+
+  private
+
+  def extract_patterns(files_data)
+    if files_data.is_a?(Hash)
+      [files_data['param1']]
+    elsif files_data.is_a?(Array)
+      files_data.map { |entry| entry['param1'] }
+    else
+      []
+    end
+  end
+end

data/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "klue-langcraft",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "klue-langcraft",
-      "version": "0.1.1",
+      "version": "0.2.0",
       "devDependencies": {
         "@klueless-js/semantic-release-rubygem": "github:klueless-js/semantic-release-rubygem",
         "@semantic-release/changelog": "^6.0.3",

data/package.json

@@ -1,6 +1,6 @@
 {
   "name": "klue-langcraft",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Domain Specific Language Crafting",
   "scripts": {
     "release": "semantic-release"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: klue-langcraft
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - David Cruwys
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-22 00:00:00.000000000 Z
+date: 2024-10-25 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: appydave-tools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: k_log
   requirement: !ruby/object:Gem::Requirement
@@ -47,27 +61,41 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/dsl_watcher.rb
 - bin/setup
+- docs/dsl-class-diagram.md
 - docs/dsl-examples.md
 - docs/dsl-rules.md
-- docs/dsl-samples/index.md
 - docs/dsl-samples/youtube-launch-optimizer-old.klue
 - docs/dsl-samples/youtube-launch-optimizer-strawberry.json
 - docs/dsl-samples/youtube-launch-optimizer-strawberry.klue
 - docs/dsl-samples/youtube-launch-optimizer.defn.klue
 - docs/dsl-samples/youtube-launch-optimizer.json
 - docs/dsl-samples/youtube-launch-optimizer.klue
+- docs/dsl-upgrade-plan.md
 - docs/project-plan/project-plan.md
 - docs/project-plan/project.drawio
 - docs/project-plan/project_done.svg
 - docs/project-plan/project_in_progress.svg
 - docs/project-plan/project_todo.svg
+- lib/base_process.rb
+- lib/dsl_folder_watcher.rb
+- lib/dsl_interpreter.rb
+- lib/dsl_process_data.rb
 - lib/klue/langcraft.rb
-- lib/klue/langcraft/-brief.md
-- lib/klue/langcraft/parser.rb
-- lib/klue/langcraft/sample_usage.rb
-- lib/klue/langcraft/tokenizer.rb
+- lib/klue/langcraft/dsl/interpreter.rb
+- lib/klue/langcraft/dsl/process_data_pipeline.rb
+- lib/klue/langcraft/dsl/process_matcher.rb
+- lib/klue/langcraft/dsl/processor_config.rb
+- lib/klue/langcraft/dsl/processors/file_collector_processor.rb
+- lib/klue/langcraft/dsl/processors/full_name_processor.rb
+- lib/klue/langcraft/dsl/processors/processor.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/-brief.md
+- lib/klue/langcraft/tokenizer-old-needs-revisit/parser.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/sample_usage.rb
+- lib/klue/langcraft/tokenizer-old-needs-revisit/tokenizer.rb
 - lib/klue/langcraft/version.rb
+- lib/process_file_collector.rb
 - package-lock.json
 - package.json
 - sig/klue/langcraft.rbs

data/docs/dsl-samples/index.md

@@ -1,4 +0,0 @@
-Printspeak DSL
-Project Plan
-Agent As Code
-Make Chapters out of my recordings using a folder and glob pattern geared to my naming convention