scout-camp 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b67e89dae365eb1464f34a19d600247a18d9698de5a8337d5cda677b1623396a
-  data.tar.gz: f5f37fce02c5a92ef3d370fd4f1e0367955bcaffc993ba5a963dca94f4ec255c
+  metadata.gz: 5535e519a380d0c2d9f9ce786c35d8ec780f58c2376901385ceb7e4bc2a9f8ef
+  data.tar.gz: 2c8ba6faccd6c9d3ebbb7951cfa6aa2b67f8ea2c374cc39a05875a1c77d2cb0e
 SHA512:
-  metadata.gz: 50d5d0ee70db54470257f49ad6b5595beb0b1d5be73df1471088fc336800a075b2d723082834f6dc6252abf1867b95323b84a3e09dd975c6be1ed2a129e2e46a
-  data.tar.gz: 6908c7789074017d48612f538878657dab8165a286ac8468fd5c243549ff1cbe599ef2868e2d5f88ef8df4f859c57aaa95c3fcd9b9603729fd662756c7faefc5
+  metadata.gz: bb1a6148578827919e2a4c4216db0c9ba69cd8666882a08bce5cf3c2e3c5e3ab56ae99bb40de806ba5b0baef93c3d2a6c76bb972a73bd3e6e82258f481760661
+  data.tar.gz: e29a4bca9bd0d91574769c57e5f99beb2317926c2ed5863448883d5a9e3ec253a108dd849984c9ed9225575988870f05ddc20a8784e03109fb2e8b533cbd4875

data/.vimproject

@@ -2,6 +2,14 @@ scout-camp=/$PWD filter="*" {
  bin=bin {
   scout-camp
  }
+ chats=chats filter="*"{
+  auth
+
+  intro
+  javascript
+  sinatra
+  unit
+ }
  lib=lib {
   scout-camp.rb
   scout=scout {
@@ -21,27 +29,58 @@ scout-camp=/$PWD filter="*" {
     sync.rb
     resource.rb
    }
+
+   render=render{
+    resource.rb
+    helpers.rb
+    engine.rb
+   }
+   render.rb
+   sinatra=sinatra{
+    entity.rb
+    workflow.rb
+
+    base=base{
+     post_processing.rb
+     parameters.rb
+
+     helpers.rb
+     assets.rb
+     headers.rb
+     session.rb
+     favicon.rb
+    }
+    base.rb
+
+    tool.rb
+
+    fragment.rb
+    auth.rb
+    htmx.rb
+    finder.rb
+    knowledge_base.rb
+   }
   }
  }
  scout_commands=scout_commands {
+  find
+  glob
   offsite
   sync
-
   terraform=terraform{
+   apply
+   destroy
+   lambda_task
    list
-   add=add{
-    relay
-    lambda
-    host
-    fargate
-   }
-   status
    outputs
-   apply
    plan
-   destroy
    remove
-   lambda_task
+   status
+   task
+   add=add{
+    lambda
+    relay
+   }
   }
  }
  share=share {
@@ -166,6 +205,8 @@ scout-camp=/$PWD filter="*" {
   aws=aws{
    lambda_function.rb
   }
+  views=views{
+  }
  }
  test=test {
   test_helper.rb

data/Rakefile

@@ -42,6 +42,11 @@ Juwelier::Tasks.new do |gem|
   #gem.add_development_dependency "simplecov", ">= 0"
 
   gem.add_runtime_dependency 'scout-essentials', '>= 0'
+  gem.add_runtime_dependency 'aws-sdk-s3', '>= 0'
+  gem.add_runtime_dependency 'sinatra', '>= 0'
+  gem.add_runtime_dependency 'omniauth', '>= 0'
+  gem.add_runtime_dependency 'mimemagic', '>= 0'
+  gem.add_runtime_dependency 'omniauth-google-oauth2', '>= 0'
 end
 Juwelier::RubygemsDotOrgTasks.new
 require 'rake/testtask'

data/VERSION

@@ -1 +1 @@
-0.1.13
+0.1.14

data/bin/scout-camp

@@ -2,6 +2,52 @@
 
 $LOAD_PATH.unshift File.join(__dir__, '../lib')
 
+if _i = ARGV.index("--log")
+  require 'scout/log'
+  log = ARGV[_i+1]
+  Log.severity = log.to_i
+  ARGV.delete "--log"
+  ARGV.delete log
+end
+
+dev_dir = nil
+if _i = ARGV.index("--dev")
+  dev_dir = ARGV[_i+1]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir.nil? 
+  _s = nil
+  ARGV.each_with_index do |s,i|
+    if s.match(/^--dev(?:=(.*))?/)
+      dev_dir = $1
+      _s = s
+      next
+    end
+  end
+  ARGV.delete _s if _s
+end
+
+if dev_dir.nil? && ENV["SCOUT_DEV"]
+  dev_dir = ENV["SCOUT_DEV"]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir
+  ['scout-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+  ['rbbt-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+end
+
 require 'scout-camp'
 
 load Scout.bin.scout.find

data/doc/terraform.md

@@ -0,0 +1,188 @@
+# Using the Terraform DLS (Domain Specific Language)
+
+This document describes how to use the Terraform DLS implemented in lib/scout/terraform_dsl to programmatically compose Terraform configurations and manage deployments from Ruby. It is written for AI agents (or developers) that will generate Ruby code to create, plan, apply and inspect Terraform deployments using the provided DSL.
+
+Where things live
+
+- Library implementation: lib/scout/terraform_dsl.rb and lib/scout/terraform_dsl/*.rb
+- Utilities and runtime: lib/scout/terraform_dsl/util.rb and lib/scout/terraform_dsl/deployment.rb
+- Example module templates: share/terraform (e.g. share/terraform/aws/*)
+
+Key classes and concepts
+
+- TerraformDSL: main class that models a planned Terraform deployment. You create a TerraformDSL instance, add modules, providers, backends, custom files, then call config to write template files into a directory.
+
+- TerraformDSL::Module: returned by add(...) and acts as a reference to a module instance inside the generated Terraform configuration. You can call arbitrary methods on it to get references to its outputs (e.g. my_module.some_output), which serialize to Terraform references like module.<instance>.<output>.
+
+- TerraformDSL::Module::Output: representation of an output reference that serializes to a module output reference in Terraform JSON form.
+
+- TerraformDSL::Deployment: helper for interacting with Terraform in a directory created by TerraformDSL#config. It wraps terraform init/plan/apply/destroy/refresh and provides helpers for reading outputs and state.
+
+- Misc helpers in TerraformDSL (util.rb):
+  - TerraformDSL.module_variables(module_dir) -> inspect variables.tf for a module
+  - TerraformDSL.module_outputs(module_dir) -> inspect output.tf for a module
+
+Common workflow (example)
+
+1) Create a TerraformDSL and add elements/modules
+
+```ruby
+require 'scout/terraform_dsl'
+
+# Create DSL (by default it uses share/terraform as modules dir)
+# You can pass a custom modules dir if needed
+dsl = TerraformDSL.new
+
+# Add a provider block (creates a provider config file)
+# Returns a Module reference if a provider module exists at modules_dir/<name>/provider
+# Example: add AWS provider configuration (set credentials/region via variables)
+dsl.provider('aws', region: 'eu-west-1')
+
+# Add a module instance using the shared templates under share/terraform
+# add(provider, module_name, variables)
+# variables[:name] defaults to "<provider>_<module_name>" but you can override
+web = dsl.add('aws', 'host', name: 'web1', ami: 'ami-01234567', instance_type: 't3.micro')
+db  = dsl.add('aws', 'host', name: 'db1', ami: 'ami-01234567', instance_type: 't3.small')
+
+# You can connect modules using module output references returned by Module instances
+# E.g. use an output from db as input to web
+# (Assuming the module exposes the output name you want)
+# web2 = dsl.add('aws', 'some_module', name: 'web2', subnet_id: db.subnet_id)
+
+# Request that specific outputs from a module become deployment outputs
+# Provide outputs in variables[:outputs] or use :outputs => 'all'
+app = dsl.add('aws','host', name: 'app1', ami: 'ami-01234567', outputs: ['aws_instance_id', 'aws_instance_ip'])
+```
+
+2) Add custom provider/backend/remote files if needed
+
+```ruby
+# Add a backend configuration (example for s3 backend)
+dsl.backend('s3', bucket: 'my-bucket', key: 'path/to/state', region: 'eu-west-1')
+
+# Add a remote data block pointing to another state
+# Returns a direct reference object you can use as a value in variables
+other_state = dsl.remote('s3', 'shared_state', bucket: 'my-bucket', region: 'eu-west-1')
+# other_state is a DirectReference object that serializes to a terraform reference
+
+# Add arbitrary terraform content (for uncommon provider configs)
+dsl.custom('provider.aws.extra', <<~TF)
+  provider "aws" {
+    # extra provider settings
+  }
+TF
+```
+
+3) Generate config files into a directory
+
+```ruby
+# Write files into a working dir (if you pass no arg, a directory based on elements digest is used)
+config_dir = dsl.config # returns the directory path (Path-like) containing .tf files
+# You can pass a path: dsl.config('/tmp/my-deploy')
+```
+
+4) Manage the terraform deployment
+
+```ruby
+# Create a Deployment helper for that directory
+deployment = TerraformDSL::Deployment.new(config_dir)
+
+# Initialize provider plugins and backend
+deployment.init
+
+# Validate if wanted
+deployment.validate
+
+# Create a plan (saves plan to main.plan)
+deployment.plan
+
+# Apply (will run plan if you didn't)
+deployment.apply
+
+# Read outputs (hash with output_name => value)
+outs = deployment.outputs
+puts outs['app1_aws_instance_ip']
+
+# Destroy the whole deployment
+deployment.destroy
+```
+
+Notes about Module references and variables
+
+- When you call add(...) you get back a TerraformDSL::Module object representing that instance.
+  - Calling arbitrary methods on that object returns a Module::Output that serializes into the Terraform reference module.<instance>.<output>. Example: web.ami_id or web.aws_instance_id depending on module output name.
+  - You can directly use these Module output references when setting variables for other modules (they will be serialized correctly into the .tf generated files).
+
+- The DSL also accepts values that are already Terraform-style strings such as "module.some_mod.some_output" — the DSL converts those into Output references when generating variable assignments.
+
+Inspection utilities
+
+- Inspect module variables (what variables a module expects):
+
+```ruby
+module_dir = Scout.share.terraform['aws']['host']
+vars = TerraformDSL.module_variables(module_dir)
+# vars is a hash: { 'ami' => {description: '...', type: 'string', default: ...}, ... }
+
+outs = TerraformDSL.module_outputs(module_dir)
+# outs is a hash of available outputs and their descriptions
+```
+
+Deployment helper details
+
+- TerraformDSL::Deployment.run(cmd)
+  - Executes terraform <cmd> and returns stdout. Raises TerraformException on non-zero exit.
+
+- TerraformDSL::Deployment.run_log(cmd, log_file=nil)
+  - Runs terraform <cmd> streaming STDERR and STDOUT into logs and raising TerraformException on error. Accepts a log_file to capture full logs.
+
+- Useful methods on Deployment objects:
+  - init, plan, apply, refresh, validate, update (init+plan+apply), destroy
+  - plan_file -> path to stored plan (main.plan)
+  - log_file -> path to log file for that work directory
+  - outputs -> reads terraform output -json and returns a Ruby hash with values
+  - element_state(element) -> shows terraform state show <element>
+  - provisioned_elements -> terraform state list
+
+Advanced: bundling and loading
+
+- You can bundle a deployment directory into a tar.gz with Deployment#bundle(file) and load it back with Deployment.load(file). The loader will extract the bundle into a working dir and refresh the state.
+
+Where to find module templates
+
+- The default module directory is Scout.share.terraform which points to share/terraform in this repository. Look under share/terraform/aws, share/terraform/ssh etc. Each module is a directory containing Terraform templates (main.tf, variables.tf, output.tf, ...).
+
+Example full script (end-to-end)
+
+```ruby
+require 'scout/terraform_dsl'
+
+# Build DSL
+dsl = TerraformDSL.new
+dsl.provider('aws', region: 'eu-west-1')
+app = dsl.add('aws', 'host', name: 'app1', ami: 'ami-01234567', instance_type: 't3.micro', outputs: ['aws_instance_id','aws_instance_ip'])
+
+# Generate files
+config_dir = dsl.config
+
+# Manage terraform
+deployment = TerraformDSL::Deployment.new(config_dir)
+deployment.init
+deployment.plan
+deployment.apply
+puts deployment.outputs # => hash of outputs
+
+# Cleanup
+deployment.destroy
+```
+
+Tips for agents generating code
+
+- Always call dsl.config before creating a Deployment. config writes the .tf files and returns the directory path used by Deployment.
+- Prefer using Module objects to pass references between modules instead of hard-coding strings. Example: db.private_ip (where db is the Module returned by add).
+- Use TerraformDSL.module_variables and module_outputs to discover what a module expects/returns when generating variables and wiring between modules.
+- Use Deployment.run_log when you want log files and streamed logs for troubleshooting.
+
+If you need more examples, look at the modules under share/terraform/aws and share/terraform/ssh for real-world templates.
+
+End of document

data/lib/scout/aws/s3.rb

@@ -24,8 +24,10 @@ module Open
     def self.claim_uri(uri)
       if Path === uri and not uri.located?
         is_s3?(uri.find)
-      else
+      elsif String === uri
         is_s3? uri
+      else
+        false
       end
     end
 
@@ -286,9 +288,9 @@ module Open
       end
 
       if Open.directory?(source)
-        cmd = "aws s3 sync #{sync_args * " "} #{source} #{target}"
+        cmd = "aws s3 sync #{sync_args * " "} '#{source}' '#{target}'"
       else
-        cmd = "aws s3 cp #{source} #{target}"
+        cmd = "aws s3 cp '#{source}' '#{target}'"
       end
       case other
       when String
@@ -296,7 +298,7 @@ module Open
       when Array
         cmd << " " << other * " "
       end
-      cmd << " && rm -Rf #{source}" if delete && ! files
+      cmd << " && rm -Rf '#{source}'" if delete && ! files
 
       if print
         cmd

data/lib/scout/offsite/resource.rb

@@ -1,12 +1,45 @@
 require 'scout/resource'
 require_relative 'sync'
 module Resource
-  def self.sync(path, map = nil, source: nil, target: nil, resource: nil, **kwargs)
-    map = 'user' if map.nil?
+  class << self
+    attr_accessor :sync_servers, :file_servers
+
+    def sync_servers
+      @sync_servers ||= begin
+                          config_file = Scout.etc.sync_servers.find_with_extension('yaml', produce: false)
+                          if config_file.exists?(produce: false)
+                            config_file.yaml
+                          else
+                            {}
+                          end
+                        end
+    end
+
+    def sync_server(resource)
+      return sync_servers[resource.to_s]
+    end
+
+    def file_servers
+      @file_servers ||= begin
+                          config_file = Scout.etc.file_servers.find_with_extension('yaml', produce: false)
+                          if config_file.exists?(produce: false)
+                            config_file.yaml
+                          else
+                            {}
+                          end
+                        end
+    end
+
+    def file_server(resource)
+      return file_servers[resource.to_s]
+    end
+
+  end
 
+  def self.sync(path, map = nil, source: nil, target: nil, resource: nil, **kwargs)
     if source
       paths = [path]
-      real_paths, identified_paths = SSHLine.locate(source, paths)
+      real_paths, identified_paths = SSHLine.locate(source, paths, map: nil)
     else
       resource = path.pkgdir if resource.nil? and path.is_a?(Path) and path.pkgdir.is_a?(Resource)
       resource = Resource.default_resource if resource.nil?
@@ -17,18 +50,90 @@ module Resource
         path = Path.setup(path, pkgdir: resource) unless path.is_a?(Path)
         real_paths = path.directory? ? path.find_all : path.glob_all
       end
-      
+
       identified_paths = real_paths.collect{|path| resource.identify(path) }
     end
 
     if target
-      target_paths = SSHLine.locate(target, identified_paths, map: map)
+      map = 'user' if map.nil?
+      target_paths, identified_paths = SSHLine.locate(target, identified_paths, map: map)
     else
       target_paths = identified_paths.collect{|p| p.find(map) }
     end
 
     real_paths.zip(target_paths).each do |source_path,target_path|
+      next if source_path.nil?
       Open.sync(source_path, target_path, kwargs.merge(source: source, target: target))
     end
   end
+
+  def self.get_from_server(path, final_path, remote_server)
+    url = File.join(remote_server, '/resource/', self.to_s, 'get_file')
+    url << "?" << Misc.hash2GET_params(:file => path, :create => false)
+
+    begin
+      @server_missing_resource_cache ||= Set.new
+      raise "Resource Not Found" if @server_missing_resource_cache.include? url
+      Net::HTTP.get_response URI(url) do |response|
+        case response
+        when Net::HTTPSuccess, Net::HTTPOK
+          Misc.sensiblewrite(final_path) do |file|
+            response.read_body do |chunk|
+              file.write chunk
+            end
+          end
+        when Net::HTTPRedirection, Net::HTTPFound
+          location = response['location']
+          Log.debug("Feching directory from: #{location}. Into: #{final_path}")
+          FileUtils.mkdir_p final_path unless File.exist? final_path
+          Misc.in_dir final_path do
+            CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
+          end
+        when Net::HTTPInternalServerError
+          @server_missing_resource_cache << url
+          raise "Resource Not Found"
+        else
+          raise "Response not understood: #{response.inspect}"
+        end
+      end
+    rescue
+      Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
+      Log.error $!.message
+      Open.rm_rf final_path if Open.exist? final_path
+      raise $!
+    end
+  end
+
+  def file_server
+    Resource.file_server(self)
+  end
+
+  def sync_server
+    Resource.sync_server(self)
+  end
+
+  alias local_produce produce
+  def produce(path, *args, **kwargs)
+    if sync_server = self.sync_server
+      begin
+        Resource.sync(Resource.default_resource.identify(path), source: sync_server)
+        return path if Open.exists?(path)
+      rescue
+        Log.exception $!
+      end
+    end
+
+    if file_server = self.file_server
+      begin
+        Resource.get_from_server(Resource.default_resource.identify(path), path.find, file_server)
+        return path if Open.exists?(path)
+      rescue
+        Log.exception $!
+      end
+    end
+
+    local_produce(path, *args, **kwargs)
+  end
+
+
 end

data/lib/scout/offsite/step.rb

@@ -5,7 +5,7 @@ require 'scout/workflow/step'
 module OffsiteStep
 
   extend Annotation
-  annotation :server, :workflow_name, :clean_id, :slurm
+  annotation :server, :workflow_name, :clean_id, :batch
 
   def inputs_directory
     @inputs_directory ||= begin
@@ -13,7 +13,7 @@ module OffsiteStep
                               file = ".scout/tmp/step_inputs/#{workflow}/#{task_name}/#{name}"
                               TmpFile.with_path do |inputs_dir|
                                 save_inputs(inputs_dir)
-                                SSHLine.rsync(inputs_dir, file, target: server, directory: true)
+                                SSHLine.rsync(inputs_dir, file, target: server, directory: inputs_dir.directory?)
                               end
                               file
                             end
@@ -30,6 +30,13 @@ module OffsiteStep
 wf = Workflow.require_workflow "#{workflow_name}";
     EOF
 
+    rec_dependencies.collect{|d| d.workflow }.compact.uniq.each do |other_workflow|
+      next if workflow_name == other_workflow.to_s
+      parts << <<~EOF.strip
+Workflow.require_workflow "#{other_workflow}" rescue nil;
+      EOF
+    end
+
     if inputs_directory
       parts << <<~EOF.strip
 job = wf.job(:#{task_name}, "#{clean_name}", :load_inputs => "#{inputs_directory}");
@@ -68,33 +75,33 @@ job = wf.job(:#{task_name}, "#{clean_name}");
     status == :done
   end
 
-  def orchestrate_slurm
+  def orchestrate_batch
     bundle_files = offsite_job_ssh <<~EOF
     require 'rbbt/hpc'
-    HPC::BATCH_MODULE = HPC.batch_system "SLURM"
-    HPC::BATCH_MODULE.orchestrate_job(job, {})
+    rules = Workflow::Orchestrator.load_rules_for_job(job)
+    Workflow::Scheduler.produce(job, rules)
     job.join
     job.bundle_files
     EOF
     SSHLine.sync(bundle_files, source: server)
-    self.load
   end
 
-
-  def exec
+  def exec(noload)
     bundle_files = offsite_job_ssh <<~EOF
-    job.run
+    Workflow.produce(job)
+    job.join
     job.bundle_files
     EOF
     SSHLine.sync(bundle_files, source: server)
-    self.load
+    noload ? self.path : self.load
   end
 
-  def run
-    if slurm
-      orchestrate_slurm
+  def run(noload=false)
+    if batch
+      orchestrate_batch
+      noload ? self.path : self.load
     else
-      exec
+      exec(noload)
     end
   end
 end