klue-langcraft 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c27050239df45678f9bf8b8527c20f033532235d023cf22bd255b06f0f48b38
-  data.tar.gz: 22e0fab109f694eb0fc501885f38e94b8e0eae55da1e66381093fc8cedbece86
+  metadata.gz: 586607de2048a4670a8cc9cd1e06dd9a2d1636aa0c9825698af2b6bac51ea24d
+  data.tar.gz: 1973c38bf614abc045c9d8e2759f984791b52da023d10441d2564bf54f99c323
 SHA512:
-  metadata.gz: 78d038c2e19671fc645c02215820345d9a89f8552869f9eb2f0d0f6769e348bef56f218ff39114bd6f8945ef44001d928b1f5a0f4896e8510ffcebe6ad9f49a3
-  data.tar.gz: 80e94051e6ad5104cba4b199b472bbea3f62bdc23903a71974fb04d9399a92beb23627581ef30c9e02a0f8a821d1a9ef48cc9cb7a19a1cd223a1af95085b9c1c
+  metadata.gz: 9c20cb42da33cc805d4ec372f6101f796b44d127724fd1a611cd2620e4614e225148fc2162ab6f0e8d9bdf2fa1909575f0ed78da93ba057b4af38415bae739bd
+  data.tar.gz: 4ebe337655eba091ec8013700734afcea0a67006bc076ac5d94e075d4210fba5ee490e43f4eb92c716e2230d5c61918ce201fecde6b6919a081bfea1a95088c9

data/.rubocop.yml

@@ -14,6 +14,7 @@ AllCops:
   Exclude:
     - ".builders/**/*"
     - "spec/samples/**/*"
+    - "lib/*" # OLD DSL INTERPRETER
 
 Metrics/BlockLength:
   Exclude:
@@ -39,8 +40,23 @@ Metrics/BlockLength:
     - shared_examples_for
     - transaction
 
+Metrics/AbcSize:
+  Exclude:
+    - bin/*
+
+
 Metrics/MethodLength:
   Max: 25
+  Exclude:
+    - bin/*
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - bin/*
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - bin/*
 
 Layout/LineLength:
   Max: 200
@@ -54,7 +70,7 @@ Lint/UnusedMethodArgument:
 Style/BlockComments:
   Enabled: false
   Include:
-    - "**/spec/*"
+    - "**/spec/**/*"
 
 # My Preferences - Start
 Metrics/ClassLength:
@@ -74,8 +90,8 @@ Style/EmptyMethod:
   Exclude:
     - "**/spec/**/*"
 Metrics/ParameterLists:
-  Exclude:
-    - "**/spec/**/*"
+  Enabled: false
+
 Layout/EmptyLineBetweenDefs:
   Exclude:
     - "**/spec/**/*"
@@ -104,3 +120,14 @@ RSpec/SpecFilePathSuffix:
 RSpec/NamedSubject:
   Exclude:
     - "**/spec/**/*"
+
+RSpec/MultipleExpectations:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/MultipleMemoizedHelpers:
+  Exclude:
+    - "**/spec/**/*"
+
+RSpec/ExampleLength:
+  Max: 20

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+# [0.2.0](https://github.com/appydave/klue-langcraft/compare/v0.1.1...v0.2.0) (2024-10-25)
+
+
+### Bug Fixes
+
+* fix ci issue ([a3bfd35](https://github.com/appydave/klue-langcraft/commit/a3bfd35371b7c0ca70b0b57d70d4541c7a4ec54b))
+* fix cop ([123070b](https://github.com/appydave/klue-langcraft/commit/123070bd1f636a81d01da469c2e5072c57384bae))
+* update dsl_interpreter ([2507045](https://github.com/appydave/klue-langcraft/commit/2507045c3f7f205603a3f899785e9c5445c4c0bb))
+
+
+### Features
+
+* refactor dsl processor ([5875dc9](https://github.com/appydave/klue-langcraft/commit/5875dc957789f113da762a6dcb55cda66bfb8a3d))
+
+## [0.1.1](https://github.com/appydave/klue-langcraft/compare/v0.1.0...v0.1.1) (2024-09-22)
+
+
+### Bug Fixes
+
+* add chatgpt conversation link ([ae5448d](https://github.com/appydave/klue-langcraft/commit/ae5448d4e23ab1fd673c788e6f30e736d82afb44))
+
 # [0.1.0](https://github.com/appydave/klue-langcraft/compare/v0.0.7...v0.1.0) (2024-09-22)
 
 

data/bin/dsl_watcher.rb

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+
+require 'klue/langcraft'
+
+BASE_PATH = ARGV[0] || '/Users/davidcruwys/dev/appydave/klueless'
+
+DSLFolderWatcher.watch(BASE_PATH)

data/bin/langcraft.rb

@@ -0,0 +1,177 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+
+require 'klue/langcraft'
+require 'optparse'
+
+# CLI class for the Klue Langcraft application.
+# This class handles command-line interactions, parsing options,
+# and executing the appropriate commands for processing files
+# or watching directories.
+class CLI
+  def initialize
+    @commands = {
+      'process' => method(:process_files),
+      'watch' => method(:watch_files)
+    }
+  end
+
+  def run
+    if ARGV.empty?
+      print_help
+      exit
+    end
+
+    command, *args = ARGV
+    if @commands.key?(command)
+      @commands[command].call(args)
+    else
+      puts "Unknown command: #{command}"
+      print_help
+    end
+  end
+
+  private
+
+  def process_files(args)
+    options = {}
+    parser = OptionParser.new do |opts|
+      opts.banner = 'Usage: langcraft process [options]'
+
+      opts.on('-i', '--input FILE', 'Input file (required)') do |file|
+        options[:input] = file
+      end
+
+      opts.on('-b', '--basic-output FILE', 'Output file') do |file|
+        options[:basic_output] = file
+      end
+
+      opts.on('-e', '--enhanced-output FILE', 'Output file') do |file|
+        options[:enhanced_output] = file
+      end
+
+      opts.on('-u', '--webhook-url URL', 'Webhook URL') do |url|
+        options[:webhook_url] = url
+      end
+
+      opts.on('-d', '--debug LEVEL', %i[none info detailed], 'Debug level (none, info, detailed)') do |level|
+        options[:debug] = level
+      end
+
+      opts.on('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end
+
+    parser.parse!(args)
+
+    if validate_options(options)
+      run_klue_runner(options)
+    else
+      print_usage(parser)
+    end
+  end
+
+  def validate_options(options)
+    if options[:input].nil?
+      puts 'Error: Input file is required.'
+      false
+    else
+      true
+    end
+  end
+
+  def run_klue_runner(options)
+    klue_runner = Klue::Langcraft::DSL::KlueRunner.new
+    klue_runner.run(
+      input_file: options[:input],
+      basic_output_file: options[:basic_output],
+      enhanced_output_file: options[:enhanced_output],
+      webhook_url: options[:webhook_url],
+      debug: options[:debug] || :none
+    )
+  rescue StandardError => e
+    puts "Error: #{e.message}"
+    exit 1
+  end
+
+  def print_usage(parser)
+    puts 'Error: Invalid or missing options.'
+    puts
+    puts parser
+    exit 1
+  end
+
+  def watch_files(args)
+    options = {
+      directories: [],
+      create_basic_json: false,
+      create_enhanced_json: false,
+      log_level: :info,
+      webhook_url: nil,
+      extensions: ['.klue'] # Default extension
+    }
+    OptionParser.new do |opts|
+      opts.banner = 'Usage: server watch [options]'
+
+      opts.on('-w', '--watch-directory DIRECTORY', 'Directory to watch (can be specified multiple times)') do |dir|
+        options[:directories] << File.expand_path(dir) # Expand the path here
+      end
+
+      opts.on('-f', '--flags TYPE', 'Set processing flags (none, basic, enhanced, all)') do |type|
+        types = type.split(',').map(&:strip).map(&:downcase)
+        options[:create_basic_json] = types.include?('basic') || types.include?('all')
+        options[:create_enhanced_json] = types.include?('enhanced') || types.include?('all')
+      end
+
+      opts.on('-l', '--log-level LEVEL', %i[none info detailed], 'Log level (none, info, detailed)') do |level|
+        options[:log_level] = level
+      end
+
+      opts.on('-u', '--webhook-url URL', 'Webhook URL') do |url|
+        options[:webhook_url] = url
+      end
+
+      opts.on('-e', '--extensions EXT1,EXT2,...', Array, 'File extensions to watch (default: .klue)') do |exts|
+        options[:extensions] = exts.map { |ext| ext.start_with?('.') ? ext : ".#{ext}" }
+      end
+
+      opts.on('-h', '--help', 'Show this message') do
+        puts opts
+        exit
+      end
+    end.parse!(args)
+
+    # If no directories were specified, use the current directory
+    options[:directories] = [Dir.pwd] if options[:directories].empty?
+
+    # Ensure all directories are expanded
+    options[:directories].map! { |dir| File.expand_path(dir) }
+
+    Klue::Langcraft::DSL::Watcher.new(
+      options[:directories],
+      create_basic_json: options[:create_basic_json],
+      create_enhanced_json: options[:create_enhanced_json],
+      log_level: options[:log_level],
+      webhook_url: options[:webhook_url],
+      extensions: options[:extensions]
+    ).start
+  end
+
+  def print_help
+    puts 'Klue Langcraft Server'
+    puts 'Usage: server [command] [options]'
+    puts ''
+    puts 'Commands:'
+    puts '  process  Process a single file'
+    puts '  watch    Watch directories for changes'
+    puts ''
+    puts "Run 'server [command] --help' for more information on a command."
+  end
+end
+
+# Run the CLI
+CLI.new.run

data/docs/dsl-class-diagram.md

@@ -0,0 +1,97 @@
+
+# DSL Class Diagram
+
+```plaintext
++----------------------------+          
+|      Klue::Langcraft::DSL   |
+|----------------------------|
+| + Interpreter               |
+| + ProcessMatcher            |
+| + ProcessorConfig           |
+| + ProcessDataPipeline       |
+| + Processors                |
++----------------------------+
+
+                |
+                v
++-----------------------------------+
+|          Interpreter              |         
+|-----------------------------------|           
+| - data: Hash                      |           
+| - processed: Boolean              |
+|-----------------------------------|           
+| + initialize()                    |
+| + process(input, output)          |
+| + method_missing()                |
+| + process_args(args, block)       |
+| + respond_to_missing?()           |
+| + to_json()                       |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|         ProcessMatcher            |         
+|-----------------------------------|           
+| + match_processors(nodes)         |           
+|-----------------------------------|
+| - traverse_nodes(node, &block)    |
+| - find_processor_for(key, value)  |
+|-----------------------------------|
+| - processor_config: ProcessorConfig|
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|      ProcessDataPipeline          |         
+|-----------------------------------|           
+| + execute(data)                   |           
+| + write_output(data, output_file) |
+|-----------------------------------|
+| - store_result(data, processor,   |
+|   processed_data)                 |
+| - calculate_index(data, processor)|
+|-----------------------------------|
+| - matcher: ProcessMatcher         |
++-----------------------------------+
+
+                |
+                v
++-----------------------------------+
+|       ProcessorConfig             |         
+|-----------------------------------|           
+| + register_processor(processor)   |           
+| + processor_for(key)              |
+| + all_processors()                |
+|-----------------------------------|
+| - processors: Hash                |
++-----------------------------------+
+
+                |
+                v
++----------------------------+          
+|         Processor           |          
+|----------------------------|           
+| - data: Hash                |
+| - key: Symbol               |
+|----------------------------|
+| + initialize(data, key)     |
+| + build_result()            |
+| + build_result_data()       |
+|----------------------------|
+| + keys() (abstract method)  |
++----------------------------+
+
+                |
+                v
++----------------------------+           +-------------------------+
+|  FileCollectorProcessor     |           |     FullNameProcessor   |
+|----------------------------|           +-------------------------+
+| (inherits Processor)        |           | (inherits Processor)    |
+|----------------------------|           +-------------------------+
+| + build_result_data()       |           | + build_result_data()   |
+| + Auto-register with        |           | + Auto-register with    |
+|   ProcessorConfig           |           |   ProcessorConfig       |
++----------------------------+           +-------------------------+
+```

data/docs/dsl-examples.md

@@ -103,3 +103,12 @@ definition :workflow do
   end
 end
 ```
+
+
+You're going to help me process a Ruby inspired DSL into JSON.
+
+There will be two parts of this conversation, part one will be about exploring the concept with a simple reflection based system that will work with any Ruby compatible structure.
+
+Part two, which I kick off after I finish part one will be to use an AST style of approach so that I can process the same concept, JavaScript or RUST.
+
+To get started, I'm going to give you some sample Ruby DSL and you're going to convert them to an equivalent ruby hash

data/docs/dsl-upgrade-plan.md

@@ -0,0 +1,266 @@
+# DSL System Expansion Plan
+
+## Overview
+
+The current system processes DSL input using an interpreter, matchers, processors, and pipelines. We need to expand the system by introducing several new features:
+
+1. **Processor Service**: A centralized service that orchestrates all operations, simplifying how the watcher and command-line tools work.
+2. **Data Saving Option**: An option to toggle whether the processed JSON data is saved to a file or not.
+3. **Endpoint for Data Delivery**: Add the capability to send the processed data to external HTTP endpoints, configurable via the command line.
+
+The changes will affect the class structure, as well as introduce new command-line options and configurations. Below is a detailed explanation of the required features and how they will be implemented.
+
+---
+
+## New Components and Changes
+
+### 1. **Processor Service**
+
+- **Purpose**: We need to introduce a new service (tentatively called `ProcessorService`) to orchestrate the different components. This class will:
+  - Call the interpreter to process the DSL input.
+  - Run the data through the processing pipeline to apply any necessary transformations.
+  - Handle the saving of the processed data to a file if configured.
+  - Send the resulting JSON data to external endpoints if URLs are provided via the command line.
+
+- **Why**: Currently, the watcher or command-line tool is responsible for calling various components (interpreter, pipeline, etc.). By introducing this service, we centralize the logic and simplify the external interfaces. The watcher will only need to call this service, and any configuration options (like file saving or sending data to an endpoint) will be handled internally by the service.
+
+- **Key Features**:
+  - **Options Handling**: The service will take in various options, such as:
+    - Whether to save the processed JSON to a file.
+    - URLs for sending data (e.g., one for simplified JSON, one for enhanced JSON).
+  - **Sequential Processing**: The service will sequentially call:
+    1. The interpreter to process the DSL.
+    2. The processing pipeline to enhance the data.
+    3. Save the data to a file (if the option is enabled).
+    4. Send the data to the endpoint (if URLs are provided).
+
+### 2. **Data Saving Option**
+
+- **Purpose**: Introduce a command-line option (`--save-to-file`) that will enable or disable saving the processed data to a file. By default, the data should flow through the system, but whether it gets written to the filesystem should be configurable.
+
+- **Why**: Not all users may want to persist the JSON data to disk. The ability to toggle this behavior will make the system more flexible.
+
+- **Details**:
+  - When the `--save-to-file` option is enabled, the processed data should be saved to a file.
+  - If the option is not provided, the data will not be saved to a file, but it will still flow through the system for other uses (e.g., sending to endpoints).
+  
+### 3. **Data Delivery to Endpoints**
+
+- **Purpose**: Add the ability to send the JSON data (either simplified or enhanced) to external HTTP endpoints. These endpoints should be configurable via command-line options (`--endpoint-simplified` and `--endpoint-enhanced`).
+
+- **Why**: The system should be able to integrate with external services by sending the resulting data to a web application. For example, the web app could receive simplified JSON for debugging and enhanced JSON for final usage. This allows seamless data transfer between the DSL processing system and external applications.
+
+- **Details**:
+  - The `ProcessorService` will check if any URLs are provided via the command line.
+  - If the URLs are provided, it will send the resulting JSON data (using HTTP POST) to those endpoints.
+  - **Two endpoints**:
+    - One for simplified JSON.
+    - One for enhanced JSON.
+  - The system should allow sending data to either or both endpoints, depending on the provided options.
+
+### 4. **Command-Line Interface (Expanded)**
+
+- The existing `CommandLineInterface` class should be expanded to handle the new options:
+  - `--save-to-file`: Toggles whether the JSON is saved to a file.
+  - `--endpoint-simplified`: Specifies the URL for sending simplified JSON.
+  - `--endpoint-enhanced`: Specifies the URL for sending enhanced JSON.
+  
+- These options should be passed to the `ProcessorService`, which will handle the actual behavior (e.g., saving the file, sending data to the endpoint).
+  
+### 5. **Watcher Integration**
+
+- The existing `Watcher` class, which uses `ListenGem` to monitor file changes, will now delegate all processing to the `ProcessorService`. When a file change is detected, the watcher should simply call the service with the necessary options.
+  
+- This means that the watcher doesn't need to know about interpreters, pipelines, or processors—it just needs to hand over the file to the `ProcessorService`, which will handle everything sequentially.
+
+---
+
+## Revised Class Structure
+
+### New Classes:
+
+1. **ProcessorService**:
+   - Central orchestrator for calling the interpreter, processing data, saving the file, and sending data to endpoints.
+   - Will take all options from the command line and watcher.
+
+2. **DataDeliveryService**:
+   - Handles sending JSON data to external HTTP endpoints.
+   - Accepts a URL and sends the provided data using HTTP POST.
+
+### Expanded Classes:
+
+1. **CommandLineInterface**:
+   - Now handles additional command-line options (`--save-to-file`, `--endpoint-simplified`, `--endpoint-enhanced`).
+   - Passes these options to the `ProcessorService`.
+
+2. **Watcher**:
+   - Instead of directly calling interpreters and pipelines, the watcher will now pass the detected file changes to the `ProcessorService` for handling.
+
+---
+
+## Command-Line Options
+
+- **New Options**:
+  - `--save-to-file`: If provided, the resulting JSON will be saved to a file. Otherwise, the data will only flow through the system.
+  - `--endpoint-simplified`: A URL for sending the simplified JSON to an external endpoint.
+  - `--endpoint-enhanced`: A URL for sending the enhanced JSON to an external endpoint.
+
+These options should be passed to the `ProcessorService`, which will then handle the appropriate behavior based on the configuration.
+
+---
+
+## Final Workflow
+
+1. **Command-Line Usage**:
+   - Users can pass options such as directories to watch, whether to save files, and URLs for endpoints.
+   
+2. **Watcher**:
+   - Monitors the specified directories for file changes and passes any detected changes to the `ProcessorService`.
+   
+3. **ProcessorService**:
+   - Orchestrates the sequence:
+     1. Calls the `Interpreter` to process the DSL input.
+     2. Runs the `ProcessDataPipeline` to enhance the data.
+     3. Saves the resulting data to a file if the `--save-to-file` option is enabled.
+     4. Sends the resulting data to the provided URL(s) if the `--endpoint-simplified` or `--endpoint-enhanced` options are specified.
+
+4. **Data Delivery**:
+   - The `DataDeliveryService` is responsible for sending the processed data to the external endpoints, handling any HTTP interactions required.
+
+
+# SAMPLE CODE, NOT TESTED BUT MIGHT BE SUITABLE
+```ruby
+
+# file: lib/klue/langcraft/dsl/data_delivery_service.rb
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Klue
+  module Langcraft
+    module DSL
+      class DataDeliveryService
+        def initialize(url)
+          @url = URI.parse(url)
+        end
+
+        def send(data)
+          http = Net::HTTP.new(@url.host, @url.port)
+          request = Net::HTTP::Post.new(@url.request_uri, { 'Content-Type' => 'application/json' })
+          request.body = data.to_json
+          response = http.request(request)
+
+          puts "Data sent to #{@url}: #{response.code} #{response.message}"
+        end
+      end
+    end
+  end
+end
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/processor_service.rb
+require_relative 'interpreter'
+require_relative 'process_data_pipeline'
+require_relative 'data_delivery_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class ProcessorService
+        def initialize(options)
+          @options = options
+          @interpreter = Interpreter.new
+          @pipeline = ProcessDataPipeline.new(ProcessMatcher.new)
+        end
+
+        def run
+          data = call_interpreter
+          enhanced_data = run_pipeline(data)
+          save_to_file(enhanced_data) if @options[:save_to_file]
+          send_to_endpoint(enhanced_data)
+        end
+
+        private
+
+        def call_interpreter
+          @interpreter.process(input: 'path_to_input_file.dsl')
+        end
+
+        def run_pipeline(data)
+          @pipeline.execute(data)
+        end
+
+        def save_to_file(data)
+          File.write('output.json', JSON.pretty_generate(data))
+        end
+
+        def send_to_endpoint(data)
+          if @options[:endpoint_simplified]
+            DataDeliveryService.new(@options[:endpoint_simplified]).send(data)
+          end
+
+          if @options[:endpoint_enhanced]
+            DataDeliveryService.new(@options[:endpoint_enhanced]).send(data)
+          end
+        end
+      end
+    end
+  end
+end
+
+```
+
+```ruby
+# file: lib/klue/langcraft/dsl/command_line_interface.rb
+require_relative 'watcher'
+require_relative 'processor_service'
+
+module Klue
+  module Langcraft
+    module DSL
+      class CommandLineInterface
+        def initialize
+          @watch_dirs = []
+          @options = {}
+        end
+
+        def start
+          parse_arguments
+          start_watcher_or_processor_service
+        end
+
+        private
+
+        def parse_arguments
+          ARGV.each_with_index do |arg, index|
+            case arg
+            when '--watch'
+              @watch_dirs << ARGV[index + 1]
+            when '--save-to-file'
+              @options[:save_to_file] = true
+            when '--endpoint-simplified'
+              @options[:endpoint_simplified] = ARGV[index + 1]
+            when '--endpoint-enhanced'
+              @options[:endpoint_enhanced] = ARGV[index + 1]
+            end
+          end
+
+          raise 'No directories specified for watching' if @watch_dirs.empty?
+        end
+
+        def start_watcher_or_processor_service
+          if @watch_dirs.any?
+            watcher = Watcher.new(@watch_dirs)
+            watcher.start
+          else
+            processor_service = ProcessorService.new(@options)
+            processor_service.run
+          end
+        end
+      end
+    end
+  end
+end
+
+```