cl-magic 1.2.10 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e26140cc95c280b62ba2fbdfba15bb2c76183ca1bab0abd27cd41489f9fe4fc
-  data.tar.gz: bb1e3783a4bddf51aff6f7b5e937095d0909f2a4c27f1973c02dfedbc0d2e1d3
+  metadata.gz: bb8d14f6e383683f5e14fe5ada204b1f78c7024b22663ff05080fddfa7d23f40
+  data.tar.gz: 2ab5fb3e7faca8be5fae4d9e2c9ea503cab062d7fc496496574ba21e4dbfdd29
 SHA512:
-  metadata.gz: c66ec59e78d1a8d1c440740c43e605a5785e33300d436f454c436597e56aceeee88259c027936d6054f4e7cf97ceb913f1cff302b039daa626f8689a1ece3f42
-  data.tar.gz: ea004d5cc57a0b1a109eb7d64991185f21d6a66009318ec24fa0303a906a4dc8e497c73e595e2a672718c6e91110c4f9a583cb7bf690b0be6545bf9e0ad5a7c8
+  metadata.gz: 49d6817adf444b4c77bf546d5d4468cd0fed212e17e27703971a8172561deb58d501ad9f03b3a24d5f00621e009ecc94018ca31f091eb1260353524b8d4752c9
+  data.tar.gz: 874d23bca143e1fd4c34ffa77da6e969fa2e4ac3fe5789e9b026712ca6bc8dfb429e969da2025f937d19cc0dff10bf2cb2a4b6753d66783f231920d6e03df69c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cl-magic (1.2.10)
+    cl-magic (1.3.1)
       activesupport
       baran
       concurrent-ruby
@@ -23,7 +23,7 @@ GEM
       tzinfo (~> 2.0)
     baran (0.1.9)
     byebug (11.1.3)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
     minitest (5.20.0)

data/README.md

@@ -24,19 +24,21 @@ ln -s $MAGIC_DIR/bin/cl /usr/local/bin
 
 ## Development
 
-If you installed the gem, you need to remove the simlink
+For local development
+
+Remove any previously installed simlink
 
 ```
 rm /usr/local/bin/cl
 ```
 
-Then sim-link to your cl-magic source code's bin script
+Then sim-link the source code instead
 
 ```
 ln -s $(pwd)/bin/cl /usr/local/bin
 ```
 
-if using `dk make-world`, you'll need to reset your world path
+NOTE: if using `dk make-world`, you'll need to reset your world path
 
 ```
 dk world set --path ./dk-world

data/bin/cl

@@ -11,7 +11,8 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
   [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
 MAGIC_DIR="$( cd -P "$( dirname "$SOURCE" )/../lib/cl/magic" >/dev/null 2>&1 && pwd )"
+MAGIC_BIN="$MAGIC_DIR/../../../bin"
 
 # make magic - with the correct version of ruby & the bundle
 cd $MAGIC_DIR
-RBENV_VERSION=3.1.2 CL_WORKING_DIR=$CL_WORKING_DIR bundle exec cl $@
+RBENV_VERSION=3.1.2 MAGIC_BIN=$MAGIC_BIN CL_WORKING_DIR=$CL_WORKING_DIR bundle exec cl $@

data/bin/common/process_pids

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -euo pipefail
+
+LOG_FILEPATHS_ARRAY=()
+PIDS_ARRAY=()
+
+process_pids () {
+
+    for pid in "${PIDS_ARRAY[@]}"
+    do
+        werr=0
+        wait $pid || werr=$?
+        if [ "$werr" != 0 ] && [ "$werr" != 127 ] ; then
+            echo "     ℹ failed - ${LOG_FILEPATHS_ARRAY[$pid]}"
+        else
+            echo "     ✔ ${LOG_FILEPATHS_ARRAY[$pid]}"
+        fi
+    done
+}

data/bin/world-scripts/build

@@ -0,0 +1,135 @@
+#!/bin/bash
+set -euo pipefail
+
+# source world constants
+source "$WORLD_DIR/_bin/common/repo_constants"
+
+# script path
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_PATH
+
+# source scripts
+source ../common/process_pids
+
+# code path
+if [ -z "$CODE_DIR" ]; thenw
+  echo "missing CODE_DIR"
+  exit 1
+fi
+code_path=$CODE_DIR
+
+# ensure shared net
+docker network create shared > /dev/null 2>&1 || true
+
+echo ""
+echo "------------------------"
+echo " make build"
+echo "------------------------"
+echo ""
+
+REPO_ARRAY=$(get_repo_array "$1")
+for repo in ${REPO_ARRAY[@]};
+do
+    repo_base_name=$(basename "${repo##*/}" .git)
+    repo_path=$code_path/$repo_base_name
+
+    echo " $repo_base_name"
+
+    if [ ! -d $repo_path ]; then
+        echo "      ℹ skip, project not cloned"
+    else
+        if [[ "$repo_base_name" =~ $(get_skip_build_regex) ]]; then
+            echo "      ℹ skip; please build manually"
+        else
+
+            cd $repo_path
+
+            # compose settings?
+            if cl dk config &> /dev/null; then
+
+                # has make init?
+                if echo $(cl dk make) | grep -q init; then
+
+                    # start: make init
+                    log_filepath="/tmp/make.build.$repo_base_name.output"
+                    echo "     ⌛ start make init"
+
+                    # set command
+                    MAKE_INIT_CMD="cl dk make init"
+
+                    # log header
+                    echo "" >> $log_filepath
+                    echo "start make init..." > $log_filepath
+                    echo $MAKE_INIT_CMD >> $log_filepath
+                    echo "" >> $log_filepath
+
+                    # go!
+                    $MAKE_INIT_CMD >> $log_filepath 2>&1  &
+
+                    # save
+                    pid=$!
+                    PIDS_ARRAY+=($pid)
+                    LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+                else
+                    echo "      ℹ skip, no 'dk make init'"
+                fi
+            else
+
+                # docker file?
+                dockerfile_filepath="$repo_path/Dockerfile"
+                if test -f "$dockerfile_filepath"; then
+                    cd $repo_path
+
+                    # arm version?
+                    ARCH=$(uname -m)
+                    if [[ ${ARCH} == 'arm64' ]]; then
+                        if test -f "$dockerfile_filepath.arm"; then
+                            echo "      ℹ found Dockerfile.arm"
+                            dockerfile_filepath="$repo_path/Dockerfile.arm"
+                        fi
+                    fi
+
+                    # start: docker build
+                    log_filepath="/tmp/make.build.$repo_base_name.output"
+                    echo "     ⌛ start docker build"
+
+                    # command
+                    DOCKER_BUILD_CMD="docker build -t dev-$repo_base_name -f $dockerfile_filepath ."
+
+                    # log header
+                    echo "" >> $log_filepath
+                    echo "start docker build..." > $log_filepath
+                    echo $DOCKER_BUILD_CMD >> $log_filepath
+                    echo "" >> $log_filepath
+
+                    # go!
+                    $DOCKER_BUILD_CMD >> $log_filepath 2>&1  &
+
+                    # save
+                    pid=$!
+                    PIDS_ARRAY+=($pid)
+                    LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+                else
+                    echo "      ℹ skip, no docker"
+                fi
+            fi
+        fi
+    fi
+    echo ""
+done
+
+if [ ! -z "${PIDS_ARRAY:-}" ]; then
+    echo ""
+    echo " waiting"
+    echo "     👀 tail -f /tmp/make.build.*.output"
+    process_pids
+else
+    echo " nothing to do.. did you 'make get' yet?"
+fi
+
+#
+# DONE
+#
+
+printf " DONE"
+echo ""

data/bin/world-scripts/down

@@ -0,0 +1,78 @@
+#!/bin/bash
+set -euo pipefail
+
+# source world constants
+source "$WORLD_DIR/_bin/common/repo_constants"
+
+# script path
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_PATH
+
+# source scripts
+source ../common/process_pids
+
+# code path
+if [ -z "$CODE_DIR" ]; then
+  echo "missing CODE_DIR"
+  exit 1
+fi
+code_path=$CODE_DIR
+
+
+log_filepaths=()
+pids=()
+
+echo ""
+echo "------------------------"
+echo " bring down services"
+echo "------------------------"
+echo ""
+
+REPO_ARRAY=$(get_repo_array "$1")
+for repo in ${REPO_ARRAY[@]};
+do
+    repo_base_name=$(basename "${repo##*/}" .git)
+    repo_path=$code_path/$repo_base_name
+
+    cd $repo_path
+
+    # compose settings?
+    if cl dk config &> /dev/null; then
+
+        # has make init?
+        if echo $(cl dk make) | grep -q down; then
+            echo " $repo_base_name"
+            dk_parts=$(get_repo_dk_parts $repo_base_name)
+
+            # run
+            log_filepath="/tmp/make.down.$repo_base_name.output"
+            echo "     ⌛ running dk down"
+            echo ""
+
+            # clear parts
+            cl dk parts clear > /dev/null 2>&1 || true
+
+            # down stack
+            cl dk make $dk_parts down > $log_filepath 2>&1 &
+
+            # save
+            pid=$!
+            PIDS_ARRAY+=($pid)
+            LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+        fi
+    fi
+done
+
+if [ ! -z "${PIDS_ARRAY:-}" ]; then
+    echo ""
+    echo " waiting"
+    echo "     👀 tail -f /tmp/make.down.*.output"
+    process_pids
+fi
+
+#
+# DONE
+#
+
+printf " DONE"
+echo ""

data/bin/world-scripts/get

@@ -0,0 +1,112 @@
+#!/bin/bash
+set -euo pipefail
+
+# source world constants
+source "$WORLD_DIR/_bin/common/repo_constants"
+
+# script path
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_PATH
+
+# source scripts
+source ../common/process_pids
+
+# code path
+if [ -z "$CODE_DIR" ]; then
+  echo "missing CODE_DIR"
+  exit 1
+fi
+code_path=$CODE_DIR
+
+# Check if the SSH agent is running
+if [ -z "$SSH_AUTH_SOCK" ]; then
+  SSH_VALID=false
+else
+  # Check if the SSH key is added to the agent
+  if ssh-add -l > /dev/null 2>&1; then
+    SSH_VALID=true
+  else
+    SSH_VALID=false
+  fi
+fi
+
+# Check if SSH_VALID is false and display an error message
+if [ "$SSH_VALID" = false ]; then
+  echo ""
+  echo "ERROR - SSH agent is not running"
+  echo ""
+  echo "This script clones multiple repositories asyncronously"
+  echo "You must have an SSH agent running with a valid key"
+  echo ""
+  echo "learn more: "
+  echo "https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent"
+  echo ""
+  exit 1
+fi
+
+echo ""
+echo "------------------------"
+echo " make get"
+echo "------------------------"
+echo ""
+
+REPO_ARRAY=$(get_repo_array "$1")
+for repo in ${REPO_ARRAY[@]};
+do
+    cd $code_path
+    repo_base_name=$(basename "${repo##*/}" .git)
+    repo_path=$code_path/$repo_base_name
+
+    echo " $repo_base_name"
+
+    log_filepath="/tmp/make.get.$repo_base_name.output"
+
+    if [ ! -d $repo_path ]; then
+        echo "     ⌛ clone repo"
+        git clone $repo >> $log_filepath 2>&1  &
+
+        # save
+        pid=$!
+        PIDS_ARRAY+=($pid)
+        LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+    else
+        # main branch?
+        cd $repo_path
+        co_branch_name=$(git rev-parse --abbrev-ref HEAD)
+        if echo $co_branch_name | grep -qE 'main|master'; then
+
+            # no local changes
+            if [[ ! `git status --porcelain` ]]; then
+
+                # then pull latest
+                echo "     ⌛ pull latest (main)"
+                git pull --rebase >> $log_filepath 2>&1  &
+
+                # save
+                pid=$!
+                PIDS_ARRAY+=($pid)
+                LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+            else
+                echo "      ℹ skip pull (repo has local changes)"
+            fi
+        else
+            echo "      ℹ skip pull ($co_branch_name)"
+        fi
+    fi
+    echo ""
+
+done
+
+if [ ! -z "${PIDS_ARRAY:-}" ]; then
+    echo ""
+    echo " waiting"
+    echo "     👀 tail -f /tmp/make.get.*.output"
+    process_pids
+fi
+
+#
+# DONE
+#
+
+printf " DONE"
+echo ""

data/bin/world-scripts/rm

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -euo pipefail
+
+# source world constants
+source "$WORLD_DIR/_bin/common/repo_constants"
+
+# script path
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_PATH
+
+# source scripts
+source ../common/process_pids
+
+# code path
+if [ -z "$CODE_DIR" ]; then
+  echo "missing CODE_DIR"
+  exit 1
+fi
+code_path=$CODE_DIR
+
+
+echo ""
+echo "🔥 WARNING: This script will delete all the $1."
+echo ""
+read -p "Type \"DELETE EVERYTHING\" to proceed: " confirmation
+echo ""
+if [ "$confirmation" != "DELETE EVERYTHING" ]; then
+    echo "Aborting!"
+    exit 1
+fi
+
+echo ""
+echo "------------------------"
+echo " make rm"
+echo "------------------------"
+echo ""
+
+REPO_ARRAY=$(get_repo_array "$1")
+for repo in ${REPO_ARRAY[@]};
+do
+    cd $code_path
+    repo_base_name=$(basename "${repo##*/}" .git)
+    repo_path=$code_path/$repo_base_name
+
+    echo " $repo_base_name"
+
+    log_filepath="/tmp/make.rm.$repo_base_name.output"
+
+    if [ -d $repo_path ]; then
+        echo "     ⌛ removing project"
+        rm -rf $repo_path >> $log_filepath 2>&1  &
+
+        # save
+        pid=$!
+        PIDS_ARRAY+=($pid)
+        LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+    else
+        echo "      ℹ skip, no local copy"
+    fi
+    echo ""
+
+done
+
+if [ ! -z "${PIDS_ARRAY:-}" ]; then
+    echo ""
+    echo " waiting"
+    echo "     👀 tail -f /tmp/make.rm.*.output"
+    process_pids
+fi
+
+#
+# DONE
+#
+
+printf " DONE"
+echo ""

data/bin/world-scripts/up

@@ -0,0 +1,83 @@
+#!/bin/bash
+set -euo pipefail
+
+# source world constants
+source "$WORLD_DIR/_bin/common/repo_constants"
+
+# script path
+SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPT_PATH
+
+# source scripts
+source ../common/process_pids
+
+# code path
+if [ -z "$CODE_DIR" ]; then
+  echo "missing CODE_DIR"
+  exit 1
+fi
+code_path=$CODE_DIR
+
+# ensure shared net
+docker network create shared > /dev/null 2>&1 || true
+
+echo ""
+echo "------------------------"
+echo " bring up compose stacks"
+echo "------------------------"
+echo ""
+
+REPO_ARRAY=$(get_repo_array "$1")
+for repo in ${REPO_ARRAY[@]};
+do
+    repo_base_name=$(basename "${repo##*/}" .git)
+    repo_path=$code_path/$repo_base_name
+
+    cd $repo_path
+
+    # compose settings?
+    if cl dk config &> /dev/null; then
+
+        # has make init?
+        if echo $(cl dk make) | grep -q up; then
+            echo " $repo_base_name"
+            dk_parts=$(get_repo_dk_parts $repo_base_name)
+
+            # run
+            log_filepath="/tmp/make.up.$repo_base_name.output"
+            echo "     ⌛ up compose stack"
+
+            # set parts
+            cl dk parts set $dk_parts > /dev/null 2>&1 || true
+
+            # up stack
+            if [[ "$repo_base_name" =~ $(get_non_async_repo_regex) ]]; then
+                echo "     👀 tail -f $log_filepath"
+                cl dk make down up > $log_filepath 2>&1
+                echo "     ✔ complete"
+                echo ""
+            else
+                echo ""
+                cl dk make down up > $log_filepath 2>&1 &
+
+                # save
+                pid=$!
+                PIDS_ARRAY+=($pid)
+                LOG_FILEPATHS_ARRAY[$pid]=$log_filepath
+            fi
+        fi
+    fi
+done
+
+if [ ! -z "${PIDS_ARRAY:-}" ]; then
+    echo ""
+    echo " waiting"
+    echo "     👀 tail -f /tmp/make.up.*.output"
+    process_pids
+fi
+
+#
+# DONE
+#
+
+echo ""

data/lib/cl/magic/cl-curl

@@ -0,0 +1,130 @@
+#!/usr/bin/env ruby
+# curl web pages by id
+require 'tty-command'
+require 'tty-prompt'
+
+require 'cl/magic/common/sub_command.rb'
+require 'cl/magic/common/common_options.rb'
+require 'cl/magic/common/logging.rb'
+require 'cl/magic/common/load_runner.rb'
+
+@logger = get_logger()
+@se_cmd_name = File.basename(__FILE__).split('-').join(' ')
+
+#
+# Features
+#
+
+def do_curls(options)
+  options[:ids].shuffle.each do |id|
+    uri = "#{options[:uri]}/#{id}"
+    status_code = `curl -s -o /dev/null -w \"%{http_code}\" #{uri}`
+    @logger.info "uri=#{uri}|status=#{status_code}"
+  end
+end
+
+def load_and_run(q, options)
+  q.load(options[:threads]) { do_curls(options) }
+  q.run
+end
+
+def run_for_durration(q, options)
+  num_seconds = 10
+  max_sleep = options[:max_sleep]
+  q.load(options[:threads]) { do_curls(options) }
+  q.run_for_durration(num_seconds, max_sleep)
+end
+
+def run_and_stagger(q, options)
+  max_sleep = options[:max_sleep]
+  q.load(options[:threads]) { do_curls(options) }
+  q.run_for_durration(max_sleep)
+end
+
+def do_work(options)
+  q = LoadRunner::Queue.new(@logger)
+  if options[:durration]
+    @logger.info "run for durration"
+    run_for_durration(q, options)
+  else
+    if options[:max_sleep] > 0
+      @logger.info "run and stagger"
+      run_and_stagger(q, options)
+    else
+      @logger.info "load and run"
+      load_and_run(q, options)
+    end
+  end
+end
+
+#
+# Options
+#
+
+options = {
+  threads: 1,
+  max_sleep: 0
+}
+global_banner = <<DOC
+
+ curl web pages by id
+
+ Usage: #{@se_cmd_name} [options]
+
+DOC
+
+global = OptionParser.new do |g|
+  g.banner = global_banner
+  add_help_and_verbose(g)
+
+  g.on("--base-uri URI", "uri to curl") do |v|
+    options[:uri] = v
+  end
+
+  g.on("--ids CSV", "comma separated list of ids") do |v|
+    options[:ids] = v.split(',')
+  end
+
+  g.on("-t", "--thread NUMBER", "number of threads (default: 1)") do |v|
+    options[:threads] = v.to_i
+  end
+
+  g.on("--max-sleep NUMBER", "max number of seconds to sleep; randomly staggers requests") do |v|
+    options[:max_sleep] = v.to_i
+  end
+
+  g.on("--durration SECONDS", "run threads for a specific amount of time") do |v|
+    options[:durration] = v.to_i
+  end
+end
+
+#
+# Run
+#
+
+@working_dir = ENV['SE_WORKING_DIR'] # passed through se-magic to here
+global.parse(ARGV)
+
+if options[:uri].nil?
+  @logger.error "missing --base-uri"
+  exit
+end
+if options[:ids].nil?
+  @logger.error "missing --ids"
+  exit
+end
+
+history_command = """#{@se_cmd_name} \\
+  --base-uri #{options[:uri]} \\
+  --ids #{options[:ids].join(',')} \\
+  --thread #{options[:threads]} \\
+  --max-sleep #{options[:max_sleep]}
+"""
+
+history_command = """#{history_command.strip} \\
+  --durration #{options[:durration]}
+""" if options[:durration]
+
+write_history(history_command)
+
+do_work(options)

data/lib/cl/magic/cl-dk

@@ -44,6 +44,10 @@ def try_print_dk_help(dk_parts_hash, dk_make_hash, args)
     puts "  - <dk-project-path>   absolute filepath to world/project directory"
     puts "  - <dk-working-path>   absolute filepath to location dk command was run from"
     puts ""
+    puts "PATHS"
+    puts "  - world-path:   #{@world_settings.get_world_path_from_settings()}"
+    puts "  - project-path: #{@world_settings.get_world_project_path()}"
+    puts "  - working-path: #{@working_dir}"
     puts "-------------------------"
   end
 end

data/lib/cl/magic/cl-dk-make

@@ -97,7 +97,7 @@ def prep_make_command(c, selected_parts)
 
   # run command
   c = interpolate_parts_into_command(c, selected_parts)
-  cmd = "cd #{@working_dir} && #{c}"
+  cmd = "cd #{@working_dir} && WORLD_DIR=#{@world_path} CODE_DIR=#{@code_path} WORKING_DIR=#{@working_dir} #{c}"
 end
 
 def interpolate_parts_into_command(cmd, selected_parts)
@@ -135,6 +135,9 @@ def do_work()
   @yaml_arg_munger = YamlArgMunger.new(@working_dir, @world_settings)
   @parts_merger = PartsMerger.new(@working_dir, @yaml_arg_munger, @help_printer, @logger)
 
+  @world_path = @world_settings.get_world_path_from_settings()
+  @code_path = @world_settings.get_code_path_from_settings()
+
   # world files
   compose_hash, dk_parts_hash, dk_make_hash = @yaml_arg_munger.get_base_compose_parts_and_make_hashes()
   if compose_hash

data/lib/cl/magic/cl-dk-make-world

@@ -1,6 +1,6 @@
-
 #!/usr/bin/env ruby
 # cli for your world
+
 require 'yaml'
 require 'tty-command'
 
@@ -94,7 +94,7 @@ def prep_make_command(c)
   @logger.wait(c.gsub("cl dk", "dk"))
 
   # run command - from world path
-  cmd = "cd #{@world_path} && WORKING_DIR=#{@working_dir} #{c}"
+  cmd = "cd #{@world_path} && WORLD_DIR=#{@world_path} CODE_DIR=#{@code_path} WORKING_DIR=#{@working_dir} #{c}"
 end
 
 #
@@ -153,5 +153,6 @@ global.parse(ARGV)
 
 # world path
 @world_path = @world_settings.get_world_path_from_settings()
+@code_path = @world_settings.get_code_path_from_settings()
 
 do_work()

data/lib/cl/magic/cl-dk-world

@@ -13,6 +13,14 @@ require_relative 'dk/world_settings'
 @logger = get_logger()
 @cl_cmd_name = File.basename(__FILE__).split('-').join(' ')
 
+def expand_path(path)
+  if path.start_with?("/") or path.start_with?("~")
+    return File.expand_path(path)
+  else
+    return File.expand_path(File.join(@working_dir, path))
+  end
+end
+
 #
 # Features
 #
@@ -20,23 +28,19 @@ require_relative 'dk/world_settings'
 def set(options, world_settings)
   world_settings_hash = world_settings.get_world_settings_hash()
 
-  # world path
-  world_path = options[:world_path]
+  # expand paths
+  world_path = expand_path(options[:world_path])
+  code_path = expand_path(options[:code_path])
 
-  if world_path.start_with?("/") or world_path.start_with?("~")
-    world_path = File.expand_path(world_path)
-  else
-    world_path = File.expand_path(File.join(@working_dir, world_path))
-  end
-
-  if TTY::Prompt.new.yes?("Set world to: #{world_path}?")
-    world_settings_hash[:world_path] = world_path
-    world_settings.save_world_settings(world_settings_hash)
-    switch_context(world_settings)
-  else
-    exit
-  end
+  # confirm
+  exit unless TTY::Prompt.new.yes?("Set world to: #{world_path}?")
+  exit unless TTY::Prompt.new.yes?("Set code to: #{code_path}?")
 
+  # set & switch context
+  world_settings_hash[:code_path] = code_path
+  world_settings_hash[:world_path] = world_path
+  world_settings.save_world_settings(world_settings_hash)
+  switch_context(world_settings)
 end
 
 def switch_context(world_settings)
@@ -97,9 +101,13 @@ global = OptionParser.new do |g|
     s.banner = dk_world_set_banner
     options[:action] = :set
 
-    s.on("-p", "--path PATH", "path to world folder") do |v|
+    s.on("-w", "--world PATH", "path to world folder") do |v|
       options[:world_path] = v
     end
+
+    s.on("-c", "--code PATH", "path to clone repos into") do |v|
+      options[:code_path] = v
+    end
   end
 
   g.subcommand 'switch_context' do |s|
@@ -122,12 +130,18 @@ world_settings = WorldSettings.new(@working_dir)
 case options[:action]
 when :set
   if options[:world_path].nil?
-    @logger.error("missing --path")
+    @logger.error("missing --world argument")
+    exit 1
+  end
+
+  if options[:code_path].nil?
+    @logger.error("missing --code argument")
     exit 1
   end
 
   history_command = """#{@cl_cmd_name} world set \\
-  --path #{options[:world_path]}
+  --world #{options[:world_path]} \
+  --code #{options[:code_path]}
 """
   write_history(history_command)
 

data/lib/cl/magic/common/load_runner.rb

@@ -0,0 +1,127 @@
+
+module LoadRunner
+
+	class Queue
+		attr_accessor :failures
+
+		def initialize(logger = nil)
+			@logger = logger if logger!=nil
+
+			# setup
+			@threads = []
+			@transactions = 0
+			@num_threads = 1
+			@failures = 0
+		end
+
+		def load(num_threads=1, &work)
+			@num_threads = num_threads
+			@work = work  # save work for new threads
+
+			# create threads
+			num_threads.times do
+
+				# do work
+				create_thread
+
+				# log
+				@logger.debug "action=create|name=thread" if @logger!=nil
+		  end
+		end
+
+		def run()
+			@start_time = Time.now
+			start_all
+			wait_until_finished
+		end
+
+		def run_and_stagger(max_sleep=10)
+			@start_time = Time.now
+			start_all { rand_sleep(max_sleep) }
+			wait_until_finished
+		end
+
+		def run_for_durration(num_seconds, max_sleep=0)
+			@start_time = Time.now
+
+			# keep them running
+			to_time = Time.now + num_seconds
+
+			# log
+			@logger.info "action=run_for_durration|until=#{to_time}" if @logger!=nil
+
+			# durration
+			while (Time.now <= to_time)
+
+				# start threads
+				@threads.each do |t|
+					break if Time.now > to_time
+
+					# restart threads
+					if (t.status=="sleep")
+						rand_sleep(max_sleep)
+						t.run
+					end
+
+					if (t.status==false or t.status == nil)
+						@transactions += 1
+						@threads.delete t
+						thread = create_thread
+						rand_sleep(max_sleep)
+						thread.run
+
+						# log
+						@logger.debug "thread_count=#{@threads.count}" if @logger!=nil
+					end
+				end
+			end
+		end
+
+		private
+
+		def create_thread()
+		  	thread = Thread.new do
+		  		@work.call
+		  	end
+		  	@threads << thread
+				sleep(0.1)
+		  	return thread
+		end
+
+		def start_all(&block)
+			@threads.each do |t|
+
+				# pausing block?
+				block
+
+				# log
+				@logger.debug "action=start|name=thread|status=#{t.status}" if @logger!=nil
+
+				# run thread
+				t.run unless t.status==false
+			end
+		end
+
+		# wait for threads to complete
+		def wait_until_finished()
+			@threads.each do |t|
+				@logger.debug "action=check_status|name=thread|status=#{t.status}" if @logger!=nil
+				if (t.status=="run" or t.status=="sleep")
+					t.join;  # wait to finish
+					@logger.debug "action=done|name=thread|status=#{t.status}" if @logger!=nil
+				end
+			end
+
+			# log
+			@logger.success "transactions=#{@transactions}|threads=#{@num_threads}|failures=#{@failures}|durration=#{Time.now - @start_time}" if @logger!=nil
+		end
+
+		# random sleep
+		def rand_sleep(max_sleep=10)
+			if max_sleep > 0
+				time_to_sleep = rand(max_sleep)
+				sleep(time_to_sleep)
+			end
+		end
+	end
+end

data/lib/cl/magic/dk/world_settings.rb

@@ -52,6 +52,14 @@ class WorldSettings
     return ""
   end
 
+  def get_code_path_from_settings()
+    world_settings = get_world_settings_hash()
+    if world_settings.key?(:code_path)
+      return File.join(world_settings[:code_path])
+    end
+    return ""
+  end
+
   private
 
   def get_repo_basename()

data/lib/cl/magic/dk/yaml_arg_munger.rb

@@ -12,11 +12,10 @@ class YamlArgMunger
     compose_hash = get_base_compose_hash()
     dk_parts_hash = {}
     dk_make_hash = {}
-    if compose_hash
-      compose_hash = merge_world_files(compose_hash, show_help=ARGV.include?("--help"))
-      dk_parts_hash = compose_hash['x-dk-parts'] ? compose_hash.delete('x-dk-parts') : {}
-      dk_make_hash = compose_hash['x-dk-make'] ? compose_hash.delete('x-dk-make') : {}
-    end
+
+    compose_hash = merge_world_files(compose_hash, show_help=ARGV.include?("--help"))
+    dk_parts_hash = compose_hash['x-dk-parts'] ? compose_hash.delete('x-dk-parts') : {}
+    dk_make_hash = compose_hash['x-dk-make'] ? compose_hash.delete('x-dk-make') : {}
     return compose_hash, dk_parts_hash, dk_make_hash
   end
 
@@ -48,11 +47,27 @@ class YamlArgMunger
   private
 
   def get_base_compose_hash()
-    cmd = "cd #{@working_dir} && docker compose config 2> /dev/null"
-    hash = YAML.load(`#{cmd}`)
-    hotfix_environment_array(hash)
+    p2_file_path=".cl-dk-config.output"
+    cmd = "cd #{@working_dir} && docker compose config --no-normalize 2> #{p2_file_path}"
+    config_output = `#{cmd}`
+    if $?.success?
+      hash = YAML.load(config_output)
+      hotfix_environment_array(hash)
+      hotfix_x_dk_settings(hash)
+      return hash ? hash : {}
+    else
+
+      # is config invalid?
+      p2_output = `cd #{@working_dir} && cat #{p2_file_path}` # get p2 output
+      `cd #{@working_dir} && rm #{p2_file_path}`              # clean up file
+      unless p2_output.include? "no configuration file provided"
+        puts "🔥 invalid docker config"
+        exit 1
+      end
 
-    return hash ? hash : {}
+      # otherwise support no base config
+      return {}
+    end
   end
 
   def merge_world_files(compose_hash, show_help=false)    
@@ -105,6 +120,42 @@ class YamlArgMunger
     end
   end
 
+  #
+  # docker compose config - alphabetizes everything
+  # so we'll only get x-dk settings from a file docker-compose.yml
+  #
+
+  def hotfix_x_dk_settings(compose_hash)
+
+    # remove x-dk settings
+    compose_hash.delete('x-dk-parts')
+    compose_hash.delete('x-dk-make')
+
+    # fetch them directly
+    supported_paths = [
+      'compose.yaml',
+      'compose.yml',
+      'docker-compose.yaml',
+      'docker-compose.yml'
+    ]
+    raw_compose_hash = nil
+    supported_paths.each do |filepath|
+      full_filepath = File.join(@working_dir, filepath)
+      if File.exist?(full_filepath)
+        begin
+          raw_compose_hash = YAML.safe_load(File.read(full_filepath))
+          break;
+        rescue StandardError => e
+          # Handle any errors during parsing
+          puts "Error parsing YAML file #{full_filepath}: #{e.message}"
+          exit 1
+        end
+      end
+    end
+    compose_hash['x-dk-parts'] = raw_compose_hash['x-dk-parts']
+    compose_hash['x-dk-make'] = raw_compose_hash['x-dk-make']
+  end
+
   #
   # our dk hash merging process
   #

data/lib/cl/magic/version.rb

@@ -2,6 +2,6 @@
 
 module Cl
   module Magic
-    VERSION = "1.2.10"
+    VERSION = "1.3.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cl-magic
 version: !ruby/object:Gem::Version
-  version: 1.2.10
+  version: 1.3.1
 platform: ruby
 authors:
 - Don Najd
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-12 00:00:00.000000000 Z
+date: 2024-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -189,9 +189,15 @@ files:
 - Gemfile.lock
 - README.md
 - bin/cl
+- bin/common/process_pids
 - bin/console
 - bin/install_gem
 - bin/setup
+- bin/world-scripts/build
+- bin/world-scripts/down
+- bin/world-scripts/get
+- bin/world-scripts/rm
+- bin/world-scripts/up
 - cl-magic.gemspec
 - lib/cl/magic.rb
 - lib/cl/magic/cl
@@ -200,6 +206,7 @@ files:
 - lib/cl/magic/cl-ai-store-jira
 - lib/cl/magic/cl-auth
 - lib/cl/magic/cl-chrome
+- lib/cl/magic/cl-curl
 - lib/cl/magic/cl-dk
 - lib/cl/magic/cl-dk-make
 - lib/cl/magic/cl-dk-make-world
@@ -233,6 +240,7 @@ files:
 - lib/cl/magic/common/gcloud.rb
 - lib/cl/magic/common/jira.rb
 - lib/cl/magic/common/kubectl.rb
+- lib/cl/magic/common/load_runner.rb
 - lib/cl/magic/common/logging.rb
 - lib/cl/magic/common/milvus.rb
 - lib/cl/magic/common/parse_and_pick.rb