klue-langcraft 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f512e176bfdc09449895ba8775adc469ef781527160585aa61a0c14b4436041
-  data.tar.gz: 1a36e20159e2d80413bd440abd42f70f5e033ac08ed14ad018b567b3c76e6ad0
+  metadata.gz: '07528e25c004b36664752d235737a9f6f521e231eb2b9e900884861e2fbeccad'
+  data.tar.gz: bd96ac7b1247f8f4fe123b4bb7923df50f1067120b1faa6e1121c6936fe400d1
 SHA512:
-  metadata.gz: a1f4d04d74426eae5675a1d45df6a6de92a23e913c1ab75fedc8762e763f4b31435d3f7f30e36c9ef0511015b201f90fe1c5b2735a94f305cdcd80f5abbe8922
-  data.tar.gz: 2f102d4c6f195477cd524c9cf919637d8cbdaa5e7ac0cbfad7ed4b7cbf5d24e65793d36838e4414e5bded37278f03c9b56f7d3096fd3db6d9a6950802ae107dc
+  metadata.gz: f270af8583b0ef01f5c290f0d69abe2de5534682169440381532fea9510c0b2b03f7c247c13dcde3fd13dc88b89fdadc3255af804138b6373513ea16a32b31d6
+  data.tar.gz: e23279553bc9f644ebe56ce796dd99cda278c10170142806e2a9ed59839a3a51a541de9b0b75082474e46f130cd6e3e5d037d2c5e9c475585280ba032d84a6d8

data/.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
   Exclude:
     - ".builders/**/*"
     - "spec/samples/**/*"
+    - "lib/*" # OLD DSL INTERPRETER
 
 Metrics/BlockLength:
   Exclude:
@@ -104,3 +105,14 @@ RSpec/SpecFilePathSuffix:
 RSpec/NamedSubject:
   Exclude:
     - "**/spec/**/*"
+
+RSpec/MultipleExpectations:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/MultipleMemoizedHelpers:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/ExampleLength:
+  Max: 20

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.1.1](https://github.com/appydave/klue-langcraft/compare/v0.1.0...v0.1.1) (2024-09-22)
+
+
+### Bug Fixes
+
+* add chatgpt conversation link ([ae5448d](https://github.com/appydave/klue-langcraft/commit/ae5448d4e23ab1fd673c788e6f30e736d82afb44))
+
+# [0.1.0](https://github.com/appydave/klue-langcraft/compare/v0.0.7...v0.1.0) (2024-09-22)
+
+
+### Features
+
+* add requirements documentation for sample parser ([c0dbe76](https://github.com/appydave/klue-langcraft/commit/c0dbe76f74d6f3d185a7eb37f507bd1e34578773))
+
 ## [0.0.7](https://github.com/appydave/klue-langcraft/compare/v0.0.6...v0.0.7) (2024-09-21)
 
 

data/bin/dsl_watcher.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+
+require 'klue/langcraft'
+
+BASE_PATH = ARGV[0] || '/Users/davidcruwys/dev/appydave/klueless'
+
+DSLFolderWatcher.watch(BASE_PATH)

data/docs/dsl-class-diagram.md

@@ -0,0 +1,97 @@
+
+# DSL Class Diagram
+
+```plaintext
++----------------------------+          
+|      Klue::Langcraft::DSL   |
+|----------------------------|
+| + Interpreter               |
+| + ProcessMatcher            |
+| + ProcessorConfig           |
+| + ProcessDataPipeline       |
+| + Processors                |
++----------------------------+
+
+                |
+                v
++-----------------------------------+
+|          Interpreter              |         
+|-----------------------------------|           
+| - data: Hash                      |           
+| - processed: Boolean              |
+|-----------------------------------|           
+| + initialize()                    |
+| + process(input, output)          |
+| + method_missing()                |
+| + process_args(args, block)       |
+| + respond_to_missing?()           |
+| + to_json()                       |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|         ProcessMatcher            |         
+|-----------------------------------|           
+| + match_processors(nodes)         |           
+|-----------------------------------|
+| - traverse_nodes(node, &block)    |
+| - find_processor_for(key, value)  |
+|-----------------------------------|
+| - processor_config: ProcessorConfig|
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|      ProcessDataPipeline          |         
+|-----------------------------------|           
+| + execute(data)                   |           
+| + write_output(data, output_file) |
+|-----------------------------------|
+| - store_result(data, processor,   |
+|   processed_data)                 |
+| - calculate_index(data, processor)|
+|-----------------------------------|
+| - matcher: ProcessMatcher         |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|       ProcessorConfig             |         
+|-----------------------------------|           
+| + register_processor(processor)   |           
+| + processor_for(key)              |
+| + all_processors()                |
+|-----------------------------------|
+| - processors: Hash                |
++-----------------------------------+
+
+                |
+                v
++----------------------------+          
+|         Processor           |          
+|----------------------------|           
+| - data: Hash                |
+| - key: Symbol               |
+|----------------------------|
+| + initialize(data, key)     |
+| + build_result()            |
+| + build_result_data()       |
+|----------------------------|
+| + keys() (abstract method)  |
++----------------------------+
+
+                |
+                v
++----------------------------+           +-------------------------+
+|  FileCollectorProcessor     |           |     FullNameProcessor   |
+|----------------------------|           +-------------------------+
+| (inherits Processor)        |           | (inherits Processor)    |
+|----------------------------|           +-------------------------+
+| + build_result_data()       |           | + build_result_data()   |
+| + Auto-register with        |           | + Auto-register with    |
+|   ProcessorConfig           |           |   ProcessorConfig       |
++----------------------------+           +-------------------------+
+```

data/docs/dsl-examples.md

@@ -103,3 +103,12 @@ definition :workflow do
   end
 end
 ```
+
+
+You're going to help me process a Ruby inspired DSL into JSON.
+
+There will be two parts of this conversation, part one will be about exploring the concept with a simple reflection based system that will work with any Ruby compatible structure.
+
+Part two, which I kick off after I finish part one will be to use an AST style of approach so that I can process the same concept, JavaScript or RUST.
+
+To get started, I'm going to give you some sample Ruby DSL and you're going to convert them to an equivalent ruby hash

data/docs/dsl-samples/youtube-launch-optimizer-strawberry.klue

@@ -12,7 +12,7 @@ workflow :youtube_launch_optimizer do
     prompt :thumbnail_idea, content: 'Suggest creative thumbnail ideas for the video.'
   end
 
-  section :title_optimization do
+  section :title_optimization, count: 3, active: true do
     step :generate_title do
       input :video_description
 

data/docs/dsl-upgrade-plan.md

@@ -0,0 +1,266 @@
+# DSL System Expansion Plan
+
+## Overview
+
+The current system processes DSL input using an interpreter, matchers, processors, and pipelines. We need to expand the system by introducing several new features:
+
+1. **Processor Service**: A centralized service that orchestrates all operations, simplifying how the watcher and command-line tools work.
+2. **Data Saving Option**: An option to toggle whether the processed JSON data is saved to a file or not.
+3. **Endpoint for Data Delivery**: Add the capability to send the processed data to external HTTP endpoints, configurable via the command line.
+
+The changes will affect the class structure, as well as introduce new command-line options and configurations. Below is a detailed explanation of the required features and how they will be implemented.
+
+---
+
+## New Components and Changes
+
+### 1. **Processor Service**
+
+- **Purpose**: We need to introduce a new service (tentatively called `ProcessorService`) to orchestrate the different components. This class will:
+  - Call the interpreter to process the DSL input.
+  - Run the data through the processing pipeline to apply any necessary transformations.
+  - Handle the saving of the processed data to a file if configured.
+  - Send the resulting JSON data to external endpoints if URLs are provided via the command line.
+
+- **Why**: Currently, the watcher or command-line tool is responsible for calling various components (interpreter, pipeline, etc.). By introducing this service, we centralize the logic and simplify the external interfaces. The watcher will only need to call this service, and any configuration options (like file saving or sending data to an endpoint) will be handled internally by the service.
+
+- **Key Features**:
+  - **Options Handling**: The service will take in various options, such as:
+    - Whether to save the processed JSON to a file.
+    - URLs for sending data (e.g., one for simplified JSON, one for enhanced JSON).
+  - **Sequential Processing**: The service will sequentially call:
+    1. The interpreter to process the DSL.
+    2. The processing pipeline to enhance the data.
+    3. Save the data to a file (if the option is enabled).
+    4. Send the data to the endpoint (if URLs are provided).
+
+### 2. **Data Saving Option**
+
+- **Purpose**: Introduce a command-line option (`--save-to-file`) that will enable or disable saving the processed data to a file. By default, the data should flow through the system, but whether it gets written to the filesystem should be configurable.
+
+- **Why**: Not all users may want to persist the JSON data to disk. The ability to toggle this behavior will make the system more flexible.
+
+- **Details**:
+  - When the `--save-to-file` option is enabled, the processed data should be saved to a file.
+  - If the option is not provided, the data will not be saved to a file, but it will still flow through the system for other uses (e.g., sending to endpoints).
+  
+### 3. **Data Delivery to Endpoints**
+
+- **Purpose**: Add the ability to send the JSON data (either simplified or enhanced) to external HTTP endpoints. These endpoints should be configurable via command-line options (`--endpoint-simplified` and `--endpoint-enhanced`).
+
+- **Why**: The system should be able to integrate with external services by sending the resulting data to a web application. For example, the web app could receive simplified JSON for debugging and enhanced JSON for final usage. This allows seamless data transfer between the DSL processing system and external applications.
+
+- **Details**:
+  - The `ProcessorService` will check if any URLs are provided via the command line.
+  - If the URLs are provided, it will send the resulting JSON data (using HTTP POST) to those endpoints.
+  - **Two endpoints**:
+    - One for simplified JSON.
+    - One for enhanced JSON.
+  - The system should allow sending data to either or both endpoints, depending on the provided options.
+
+### 4. **Command-Line Interface (Expanded)**
+
+- The existing `CommandLineInterface` class should be expanded to handle the new options:
+  - `--save-to-file`: Toggles whether the JSON is saved to a file.
+  - `--endpoint-simplified`: Specifies the URL for sending simplified JSON.
+  - `--endpoint-enhanced`: Specifies the URL for sending enhanced JSON.
+  
+- These options should be passed to the `ProcessorService`, which will handle the actual behavior (e.g., saving the file, sending data to the endpoint).
+  
+### 5. **Watcher Integration**
+
+- The existing `Watcher` class, which uses `ListenGem` to monitor file changes, will now delegate all processing to the `ProcessorService`. When a file change is detected, the watcher should simply call the service with the necessary options.
+  
+- This means that the watcher doesn't need to know about interpreters, pipelines, or processors—it just needs to hand over the file to the `ProcessorService`, which will handle everything sequentially.
+
+---
+
+## Revised Class Structure
+
+### New Classes:
+
+1. **ProcessorService**:
+   - Central orchestrator for calling the interpreter, processing data, saving the file, and sending data to endpoints.
+   - Will take all options from the command line and watcher.
+
+2. **DataDeliveryService**:
+   - Handles sending JSON data to external HTTP endpoints.
+   - Accepts a URL and sends the provided data using HTTP POST.
+
+### Expanded Classes:
+
+1. **CommandLineInterface**:
+   - Now handles additional command-line options (`--save-to-file`, `--endpoint-simplified`, `--endpoint-enhanced`).
+   - Passes these options to the `ProcessorService`.
+
+2. **Watcher**:
+   - Instead of directly calling interpreters and pipelines, the watcher will now pass the detected file changes to the `ProcessorService` for handling.
+
+---
+
+## Command-Line Options
+
+- **New Options**:
+  - `--save-to-file`: If provided, the resulting JSON will be saved to a file. Otherwise, the data will only flow through the system.
+  - `--endpoint-simplified`: A URL for sending the simplified JSON to an external endpoint.
+  - `--endpoint-enhanced`: A URL for sending the enhanced JSON to an external endpoint.
+
+These options should be passed to the `ProcessorService`, which will then handle the appropriate behavior based on the configuration.
+
+---
+
+## Final Workflow
+
+1. **Command-Line Usage**:
+   - Users can pass options such as directories to watch, whether to save files, and URLs for endpoints.
+   
+2. **Watcher**:
+   - Monitors the specified directories for file changes and passes any detected changes to the `ProcessorService`.
+   
+3. **ProcessorService**:
+   - Orchestrates the sequence:
+     1. Calls the `Interpreter` to process the DSL input.
+     2. Runs the `ProcessDataPipeline` to enhance the data.
+     3. Saves the resulting data to a file if the `--save-to-file` option is enabled.
+     4. Sends the resulting data to the provided URL(s) if the `--endpoint-simplified` or `--endpoint-enhanced` options are specified.
+
+4. **Data Delivery**:
+   - The `DataDeliveryService` is responsible for sending the processed data to the external endpoints, handling any HTTP interactions required.
+
+
+# SAMPLE CODE, NOT TESTED BUT MIGHT BE SUITABLE
+```ruby
+
+# file: lib/klue/langcraft/dsl/data_delivery_service.rb
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Klue
+  module Langcraft
+    module DSL
+      class DataDeliveryService
+        def initialize(url)
+          @url = URI.parse(url)
+        end
+
+        def send(data)
+          http = Net::HTTP.new(@url.host, @url.port)
+          request = Net::HTTP::Post.new(@url.request_uri, { 'Content-Type' => 'application/json' })
+          request.body = data.to_json
+          response = http.request(request)
+
+          puts "Data sent to #{@url}: #{response.code} #{response.message}"
+        end
+      end
+    end
+  end
+end
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/processor_service.rb
+require_relative 'interpreter'
+require_relative 'process_data_pipeline'
+require_relative 'data_delivery_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class ProcessorService
+        def initialize(options)
+          @options = options
+          @interpreter = Interpreter.new
+          @pipeline = ProcessDataPipeline.new(ProcessMatcher.new)
+        end
+
+        def run
+          data = call_interpreter
+          enhanced_data = run_pipeline(data)
+          save_to_file(enhanced_data) if @options[:save_to_file]
+          send_to_endpoint(enhanced_data)
+        end
+
+        private
+
+        def call_interpreter
+          @interpreter.process(input: 'path_to_input_file.dsl')
+        end
+
+        def run_pipeline(data)
+          @pipeline.execute(data)
+        end
+
+        def save_to_file(data)
+          File.write('output.json', JSON.pretty_generate(data))
+        end
+
+        def send_to_endpoint(data)
+          if @options[:endpoint_simplified]
+            DataDeliveryService.new(@options[:endpoint_simplified]).send(data)
+          end
+
+          if @options[:endpoint_enhanced]
+            DataDeliveryService.new(@options[:endpoint_enhanced]).send(data)
+          end
+        end
+      end
+    end
+  end
+end
+
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/command_line_interface.rb
+require_relative 'watcher'
+require_relative 'processor_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class CommandLineInterface
+        def initialize
+          @watch_dirs = []
+          @options = {}
+        end
+
+        def start
+          parse_arguments
+          start_watcher_or_processor_service
+        end
+
+        private
+
+        def parse_arguments
+          ARGV.each_with_index do |arg, index|
+            case arg
+            when '--watch'
+              @watch_dirs << ARGV[index + 1]
+            when '--save-to-file'
+              @options[:save_to_file] = true
+            when '--endpoint-simplified'
+              @options[:endpoint_simplified] = ARGV[index + 1]
+            when '--endpoint-enhanced'
+              @options[:endpoint_enhanced] = ARGV[index + 1]
+            end
+          end
+
+          raise 'No directories specified for watching' if @watch_dirs.empty?
+        end
+
+        def start_watcher_or_processor_service
+          if @watch_dirs.any?
+            watcher = Watcher.new(@watch_dirs)
+            watcher.start
+          else
+            processor_service = ProcessorService.new(@options)
+            processor_service.run
+          end
+        end
+      end
+    end
+  end
+end
+
+```

data/lib/base_process.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# file: lib/base_process.rb
+
+class BaseProcess
+  attr_reader :key
+
+  def initialize(key)
+    @key = key
+  end
+
+  def deep_match(input, predicate)
+    matches = []
+
+    # If the current input is a Hash, iterate over each key-value pair
+    if input.is_a?(Hash)
+
+      input.each do |key, value|
+
+        # If the value matches the predicate, add it to matches
+        if predicate.call(key, value)
+          matches << value
+        end
+
+        # Continue searching deeper within the value
+        matches.concat(deep_match(value, predicate))
+      end
+
+    # If the input is an Array, iterate over each item
+    elsif input.is_a?(Array)
+
+      input.each do |item|
+
+        # Continue searching within each item of the array
+        matches.concat(deep_match(item, predicate))
+      end
+    end
+
+    matches
+  end
+end

data/lib/dsl_folder_watcher.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+class DSLFolderWatcher
+  def self.watch(folder_path)
+    puts "Watching: #{folder_path}"
+    listener = Listen.to(folder_path) do |modified, added, _removed|
+      changes = (modified + added).uniq
+
+      # DEBOUNCE CURRENTLY NOT WORKING
+      # debounce_map = {}
+      # debounce_interval = 1 # seconds
+
+      changes.each do |file_path|
+        next unless File.extname(file_path) == '.klue'
+
+        puts file_path
+
+        # debounce_map[file_path] ||= Time.now
+        # next unless Time.now - debounce_map[file_path] >= debounce_interval
+
+        # debounce_map[file_path] = Time.now
+
+        base_name = file_path.gsub(/\.klue$/, '')
+        input_file = "#{base_name}.klue"
+        output_file = "#{base_name}.json"
+
+        interpreter = DSLInterpreter.new
+        if interpreter.process('', input_file, output_file)
+          # Process the JSON data to add 'process-data' details
+          dsl_processor = DSLProcessData.new
+          dsl_processor.process('', output_file, output_file)
+          # SKIP EXTEND FILE FOR NOW AND REWRITE THE OUTPUTFILE
+          # dsl_processor.process('', output_file, extended_output_file)
+
+          # interpreter.send_to_endpoint
+        else
+          puts 'Skipping further processing due to errors in DSL interpretation.'
+        end
+      end
+
+      # Remove old entries from debounce_map to prevent memory bloat
+      # debounce_map.each_key do |key|
+      #   debounce_map.delete(key) if Time.now - debounce_map[key] > debounce_interval * 2
+      # end
+    end
+    listener.start
+    puts "Wait for changes: #{folder_path}"
+    sleep
+  end
+end

data/lib/dsl_interpreter.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+# ChatReferences:
+# - https://chatgpt.com/c/67064770-d524-8002-8344-3091e895d150
+# - https://chatgpt.com/c/6706289c-9b9c-8002-86e3-f9198c1c608a
+# - https://chatgpt.com/c/670dcd34-5dbc-8002-ad7a-d4df54a6a2e0
+#
+class DSLInterpreter
+  def initialize
+    @data = {}
+  end
+
+  # Capturing top-level DSL methods
+  def method_missing(method_name, *args, &block)
+    key = method_name
+    value = process_args(args, block)
+
+    # Append key-value to the current context of @data
+    if @data[key]
+      @data[key] = [@data[key]] unless @data[key].is_a?(Array)
+      @data[key] << value
+    else
+      @data[key] = value
+    end
+  end
+
+  # A method to handle parameters and nested blocks
+  def process_args(args, block)
+    data = {}
+
+    # Handling positional and named parameters separately
+    positional_args = []
+    named_args = {}
+
+    args.each do |arg|
+      if arg.is_a?(Hash)
+        named_args.merge!(arg)
+      else
+        positional_args << arg
+      end
+    end
+
+    # Assign positional parameters generically
+    positional_args.each_with_index do |arg, index|
+      data[:"param#{index + 1}"] = arg
+    end
+
+    # Merge named parameters directly
+    data.merge!(named_args)
+
+    # Handling a nested block
+    if block
+      interpreter = DSLInterpreter.new
+      interpreter.instance_eval(&block)
+      data.merge!(interpreter.data)
+    end
+
+    data.empty? ? nil : data
+  end
+
+  # To access data after interpreting
+  attr_reader :data
+
+  # Reading file and evaluating as Ruby
+  def process(base_path, input_file, output_file)
+    file_path = File.join(base_path, input_file)
+    content = File.read(file_path)
+
+    # begin
+    instance_eval(content)
+    # rescue SyntaxError => e
+    #   puts "Syntax error in DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at line: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # rescue StandardError => e
+    #   puts "Error processing DSL file: #{input_file}"
+    #   puts "Error message: #{e.message}"
+    #   puts "Error occurred at: #{e.backtrace.first}"
+    #   return false  # Indicate that processing failed
+    # end
+
+    output_path = File.join(base_path, output_file)
+    File.write(output_path, JSON.pretty_generate(to_hash))
+    true # Indicate that processing succeeded
+  end
+
+  # Convert to hash or JSON as required
+  def to_hash
+    @data
+  end
+
+  def to_json(*_args)
+    @data.to_json
+  end
+
+  # Method to send data to an endpoint
+  def send_to_endpoint
+    root_key = @data.keys.first
+    action_type = root_key.to_s
+
+    uri = URI.parse("http://localhost:4567/dsl/#{action_type}")
+    http = Net::HTTP.new(uri.host, uri.port)
+    request = Net::HTTP::Post.new(uri.path, { 'Content-Type' => 'application/json' })
+    payload = { action_type: action_type, data: @data }
+    request.body = payload.to_json
+
+    response = http.request(request)
+    puts "Response: #{response.code} - #{response.message}"
+    puts "Endpoint: #{uri}"
+  end
+end

data/lib/dsl_process_data.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class DSLProcessData
+  PROCESSORS = [{ file_collector: ProcessFileCollector }].freeze
+
+  # Method to process the JSON file after initial evaluation
+  def process(base_path, input_file, output_file)
+    json_file_path = File.join(base_path, input_file)
+    data = JSON.parse(File.read(json_file_path))
+
+    # Loop through the processors and execute matching ones
+    PROCESSORS.each do |processor_entry|
+      key, processor_class = processor_entry.first
+      processor = processor_class.new(key)
+
+      next unless processor.match?(data)
+
+      result = processor.execute(data)
+
+      data['process-data'] ||= {}
+
+      result.each do |key, result|
+        data['process-data'][key.to_s] = result unless result.empty?
+      end
+    end
+
+    # Write the updated JSON data to an extended file
+    extended_output_file = File.join(base_path, output_file)
+    File.write(extended_output_file, JSON.pretty_generate(data))
+  end
+end