scout-camp 0.1.13 → 0.1.14

data/lib/scout/sinatra/fragment.rb

@@ -0,0 +1,275 @@
+require_relative 'render/parameters'
+
+module ScoutRenderHelpers
+  def fragment(link = nil, &block)
+    fragment_code, link = [link.to_s, nil] if link and not link.to_s[0] == '<'
+    text = fragment_code if fragment_code
+
+    if block_given?
+      if defined? @step and (_cache_type == :asynchronous or _cache_type == :async)
+
+        fragment_code ||= (rand * 100000).to_i.to_s
+        fragment_file = @step.file(fragment_code)
+        pid_file = fragment_file + '.pid'
+
+        pid = @step.child{
+          begin
+            class << @step
+              def status=(message)
+                nil
+              end
+            end
+            Log.low("Fragment started: #{ fragment_file } - #{Process.pid}")
+            res = block.call fragment_file
+            Log.low("Fragment writing: #{ fragment_file } - #{Process.pid}")
+            Open.write(fragment_file, res) unless res.nil?
+            Log.low("Fragment done: #{ fragment_file } - #{Process.pid}")
+          rescue Exception
+            Open.write(fragment_file + '.error', [$!.class.to_s, $!.message] * ": ")
+            Open.write(fragment_file + '.backtrace', $!.backtrace * "\n") if $!.backtrace
+            Log.error("Error in fragment: #{ fragment_file }")
+            Log.exception $!
+            Open.rm pid_file if Open.exists? pid_file
+            Kernel.exit! -1
+          ensure
+            Open.rm pid_file if Open.exists? pid_file
+          end
+          Kernel.exit! 0
+        }
+        Open.write(pid_file, pid.to_s)
+
+        url = fullpath 
+        url = remove_GET_param(url, "_update")
+        url = remove_GET_param(url, "_")
+
+        fragment_url = add_GET_param(url, "_fragment", fragment_code)
+        if link.nil?
+          html_tag('a', "", :href => fragment_url, :class => 'fragment', "data-text" => text)
+        else
+          if FalseClass === link
+            return fragment_code
+          elsif TrueClass === link
+            return fragment_url
+          elsif link =~ / href=/
+            link = link.sub(/ href=('|")/," href='#{fragment_url}'")
+          else
+            link = link.sub(/<a /,"<a href='#{fragment_url}' ")
+          end
+
+          if text
+            link.sub(/<a /,"<a data-text='#{text}' ")
+          else
+            link
+          end
+        end
+      else
+        block.call
+      end
+    else
+      if link =~ / class=/
+        link = link.sub(/ class=('|")/,' class=\1fragment ')
+      else
+        link = link.sub(/<a /,'<a class="fragment" ')
+      end
+      
+      if text
+        link.sub(/<a /,"<a data-text='#{text}' ")
+      else
+        link
+      end
+    end
+  end
+end
+
+module SinatraScoutFragment
+  def self.registered(app)
+    app.helpers do
+      # keep a small param consumer (safe)
+      def process_fragment(fragment_code)
+        fragment_file = @step.file(fragment_code)
+        if Open.exists?(fragment_file)
+          case _format.to_s
+          when "query-entity"
+            tsv, table_options = load_tsv(fragment_file, true)
+            begin
+              res = tsv[@entity].to_json
+              content_type "application/json" 
+            rescue
+              res = nil.to_json
+            end
+            halt 200, res 
+          when "query-entity-field"
+            tsv, table_options = load_tsv(fragment_file, true)
+            begin
+              res = tsv[@entity]
+              res = [res] if tsv.type == :single or tsv.type == :flat
+            rescue
+              res = nil.to_json
+            end
+
+            fields = tsv.fields
+            content_type "application/json" 
+            hash = {}
+            fields.each_with_index do |f,i|
+              hash[f] = res.nil? ? nil : res[i]
+            end
+
+            halt 200, hash.to_json 
+          when "table"
+            html_halt tsv2html(fragment_file)
+          when "json"
+            content_type "application/json" 
+            halt 200, tsv_process(load_tsv(fragment_file).first).to_json
+          when "tsv"
+            content_type "text/tab-separated-values"
+            halt 200, tsv_process(load_tsv(fragment_file).first).to_s
+          when "values"
+            tsv = tsv_process(load_tsv(fragment_file).first)
+            list = tsv.values.flatten
+            content_type "application/json" 
+            halt 200, list.compact.to_json
+          when "entities"
+            raw_tsv, tsv_options = load_tsv(fragment_file)
+            tsv = tsv_process(raw_tsv)
+
+            list = tsv.values.flatten
+            list = tsv.prepare_entity(list, tsv.fields.first, tsv.entity_options)
+            type = list.annotation_types.last
+            list_id = "List of #{type} in table #{ @fragment }"
+            list_id << " (#{ @filter })" if @filter
+            Entity::List.save_list(type.to_s, list_id, list, user)
+            url =  Entity::REST.entity_list_url(list_id, type)
+            url = url + '?_layout=false' unless @layout
+            url = escape_url(url)
+            redirect to(url)
+          when "map"
+            raw_tsv, tsv_options = load_tsv(fragment_file)
+            raw_tsv.unnamed = true
+            tsv = tsv_process(raw_tsv)
+
+            field = tsv.key_field
+            column = tsv.fields.first
+
+            if tsv.entity_templates[field] 
+              type = tsv.entity_templates[field].annotation_types.first
+            else
+              type = [Entity.formats[field]].compact.first || field
+            end
+
+            map_id = "Map #{type}-#{column} in #{ @fragment }"
+            map_id << " (#{ @filter.gsub(';','|') })" if @filter
+            Entity::Map.save_map(type.to_s, column, map_id, tsv, user)
+            url = Entity::REST.entity_map_url(map_id, type, column)
+            url = url + '?_layout=false' unless @layout
+            url = escape_url(url)
+            redirect to(url)
+          when "excel"
+            require 'rbbt/tsv/excel'
+            tsv, tsv_options = load_tsv(fragment_file)
+            tsv = tsv_process(tsv)
+            data = nil
+            excel_file = TmpFile.tmp_file
+            tsv.excel(excel_file, :sort_by => @excel_sort_by, :sort_by_cast => @excel_sort_by_cast, :name => true, :remove_links => true)
+            name = tsv_options[:table_id]
+            name ||= "rbbt-table"
+            name = name.sub(/\s/,'_')
+            name = name.sub('.tsv','')
+            send_file excel_file, :type => 'application/vnd.ms-excel', :filename => "#{name}.xls"
+          when "heatmap"
+            require 'rbbt/util/R'
+            tsv, tsv_options = load_tsv(fragment_file)
+            content_type "text/html"
+            data = nil
+            png_file = TmpFile.tmp_file + '.png'
+            width = tsv.fields.length * 10 + 500
+            height = tsv.size * 10 + 500
+            width = 10000 if width > 10000
+            height = 10000 if height > 10000
+            tsv.R <<-EOF
+rbbt.pheatmap(file='#{png_file}', data, width=#{width}, height=#{height})
+data = NULL
+            EOF
+            send_file png_file, :type => 'image/png', :filename => fragment_file + ".heatmap.png"
+          when 'binary'
+            send_file fragment_file, :type => 'application/octet-stream'
+          else
+            require 'mimemagic'
+            mime = nil
+            Open.open(fragment_file) do |io|
+              begin
+                mime = MimeMagic.by_path(fragment_file) 
+                if mime.nil?
+                  io.rewind
+                  mime = MimeMagic.by_magic(io) 
+                end
+                if mime.nil?
+                  io.rewind if IO === io
+                  mime = "text/tab-separated-values" if io.gets =~ /^#/ and io.gets.include? "\t"
+                end
+              rescue Exception
+                Log.exception $!
+              end
+            end
+
+            if mime.nil?
+              txt = Open.read(fragment_file)
+
+              if txt =~ /<([^> ]+)[^>]*>.*?<\/\1>/m
+                mime = "text/html"
+              else
+                begin
+                  JSON.parse(txt)
+                  mime = "application/json"
+                rescue
+                end
+              end
+            else
+              txt = nil
+            end
+
+            if mime
+              content_type mime 
+            else
+              content_type "text/plain"
+            end
+
+            if mime && mime.to_s.include?("text/html")
+              html_halt txt || Open.read(fragment_file)
+            else
+              if File.exist?(fragment_file)
+                send_file fragment_file
+              else
+                halt 200, Open.read(fragment_file)
+              end
+            end
+          end
+        elsif Open.exists?(fragment_file + '.error') 
+          klass, _sep, message = Open.read(fragment_file + '.error').partition(": ")
+          backtrace = Open.read(fragment_file + '.backtrace').split("\n")
+          exception =  Kernel.const_get(klass).new message || "no message"
+          exception.set_backtrace backtrace
+          raise exception
+          #halt 500, html_tag(:span, File.read(fragment_file + '.error'), :class => "message") + 
+          #  html_tag(:ul, File.read(fragment_file + '.backtrace').split("\n").collect{|l| html_tag(:li, l)} * "\n", :class => "backtrace") 
+        elsif Open.exists?(fragment_file + '.pid') 
+          pid = Open.read(fragment_file + '.pid')
+          if Misc.pid_exists?(pid.to_i)
+            halt 202, "Fragment not completed"
+          else
+            halt 500, "Fragment aborted"
+          end
+        else
+          halt 500, "Fragment not completed and no pid file"
+        end
+      end
+    end
+  end
+end
+
+SinatraScoutParameters.register_common_parameter(:_fragment, :string)
+
+SinatraScoutPostProcessing.register_post_processing do |step|
+  if _fragment
+    process_fragment(_fragment)
+  end
+end

data/lib/scout/sinatra/htmx.rb

@@ -0,0 +1,68 @@
+
+module SinatraScoutHTMX
+  
+  def self.registered(app)
+    app.before do
+      headers "X-Frame-Options" => "DENY"
+    end
+
+    app.after do
+      triggers = []
+
+      status = case response.status
+               when 200
+                 ['done', 'complete']
+               when 500
+                 ['error', 'complete']
+               when 202
+                 ['running']
+               else
+                 []
+               end
+
+      if defined?(entity_type) && entity_type
+        entity = (splat*"/").gsub(/\s/,'_')
+        triggers << 'entity' 
+        triggers << entity_type
+        triggers << entity 
+        triggers << [entity_type, entity]*'_'
+      end
+
+      if defined?(entity_property) && entity_property
+        triggers << 'entity_property' 
+        triggers << [entity_type, entity_property]*'_'
+        triggers << [entity, entity_property]*'_'
+        triggers << [entity_type, entity, entity_property]*'_'
+        triggers << entity_property
+      end
+
+      if defined?(entity_action) && entity_action
+        triggers << 'entity_action' 
+        triggers << [entity_type, entity_action]*'_'
+        triggers << [entity, entity_action]*'_'
+        triggers << [entity_type, entity, entity_action]*'_'
+        triggers << entity_action
+      end
+
+      if defined?(task_name) && task_name
+        triggers << 'task' 
+        triggers << task_name
+        triggers << [workflow, task_name]*"_"
+      end
+
+      if defined?(preferences_changed) && preferences_changed.any?
+        triggers << ['preference'] 
+        preferences_changed.each do |preference|
+          triggers << [preference] 
+          triggers << ['preference_' + preference] 
+        end
+      end
+
+      triggers += triggers.collect{|t| status.collect{|s| [t, s]*"_" }}.flatten
+
+      triggers << status
+
+      headers['HX-Trigger'] = triggers * ", "
+    end
+  end
+end

data/lib/scout/sinatra/knowledge_base.rb

@@ -0,0 +1,14 @@
+require 'scout/knowledge_base'
+
+module SinatraScoutKnowledgeBase
+  def self.registered(app)
+    # helpers for REST-style responses & params
+    app.set :knowledge_base, KnowledgeBase.load(:default)
+    
+    app.helpers do
+      def knowledge_base
+        settings.knowledge_base
+      end
+    end
+  end
+end

data/lib/scout/sinatra/tool.rb

@@ -0,0 +1,11 @@
+module SinatraScoutTool
+  def self.registered(app)
+    app.helpers do
+      def tool(name, params = {})
+        record_js "/js/tool/#{name}.js" if ScoutRender.exists?("public/js/tool/#{name}.js")
+        record_css "/css/tool/#{name}.css" if ScoutRender.exists?("public/css/#{name}.css")
+        render_partial("tool/#{name}", params)
+      end
+    end
+  end
+end

data/lib/scout/sinatra/workflow.rb

@@ -0,0 +1,129 @@
+require 'sinatra'
+require_relative 'base'
+require_relative 'knowledge_base'
+
+module SinatraScoutWorkflow
+  def self.registered(app)
+    app.register SinatraScoutKnowledgeBase
+
+    app.set :registered_workflows, {}
+
+    # helpers for REST-style responses & params
+    app.helpers do
+      # prepare task inputs: read keys from provided task_info[:inputs]
+      def consume_task_parameters_for(workflow, task, params_hash)
+        task_info = workflow.task_info(task.to_sym)
+        inputs = {}
+        task_info[:inputs].each do |k|
+          v = params_hash[k.to_s]
+          inputs[k.to_sym] = v unless v.nil?
+          # accept file parts named "#{k}__param_file" (Rack file upload hash)
+          if params_hash["#{k}__param_file"]
+            inputs["#{k}__param_file"] = params_hash["#{k}__param_file"]
+          end
+        end
+        inputs
+      end
+
+      def workflow_render(workflow, template, params = {})
+        params[:workflow] = workflow
+        case template
+        when :tasks
+          render_template('tasks', params)
+        when Symbol, String
+          render_template("#{workflow}/#{template}", params.merge(task: template))
+        when Step
+          job = template
+          task_name = job.task_name
+          render_template("#{workflow}/#{task_name}", params.merge(task: task_name, job: job))
+        end
+      end
+
+      def workflow_template(workflow, task, type = :job)
+        workflow_name = Workflow === workflow ? workflow.name : workflow.to_s
+        file = File.join(workflow_name, task.to_s, type.to_s)
+        return file if ScoutRender.exists?(file)
+
+        workflow_name = Workflow === workflow ? workflow.name : workflow.to_s
+        file = File.join(workflow_name, task.to_s)
+        return file if ScoutRender.exists?(file)
+
+        file = File.join(workflow_name, type.to_s)
+        return file if ScoutRender.exists?(file)
+
+        file = File.join(type.to_s)
+        return file if ScoutRender.exists?(file)
+
+        raise TemplateNotFoundException, "Workflow template not fond for workflow #{workflow_name} task #{task} type #{type}"
+      end
+
+      def render_job(workflow, job, params = {})
+        task = job.task_name
+        template = workflow_template(workflow, task)
+        render_template(template, params.merge(job: job, task: task, workflow: workflow))
+      end
+
+      def job_url(job)
+        workflow = job.workflow
+        task = job.task_name
+        name = job.name
+        workflow_name = Workflow === workflow ? workflow.name : workflow.to_s
+
+        '/' + [workflow_name, task, name] * "/"
+      end
+
+      def workflow_name
+        return nil unless task_name
+        return fullpath.split("/")[1]
+      end
+
+      def workflow
+        return nil unless workflow_name
+        Kernel.const_get workflow_name
+      end
+
+      def job_name
+        splat * "/"
+      end
+
+      def job
+        return nil unless task_name
+        return nil unless job_name
+        job = workflow.load_job task_name, job_name
+      end
+    end
+
+    # expose method to register a workflow at runtime
+    app.define_singleton_method(:add_workflow) do |workflow, opts = {}|
+      name = workflow.to_s.split('::').last
+      settings.registered_workflows[name] = workflow
+      settings.knowledge_base = workflow.knowledge_base if workflow.knowledge_base
+      ScoutRender.prepend_path name, workflow.libdir
+
+      # GET /<workflow> - exports
+      app.get "/#{name}/:task_name/*" do
+        job = workflow.load_job task_name, job_name
+
+        case _format
+        when :info
+          serve_step_info job
+        when :json
+          serve_step_json job
+        when :raw
+          serve_step_raw job
+        else
+          serve_step job, _layout, params do
+            begin
+              render_job workflow, job, params
+            rescue TemplateNotFoundException
+              halt 200, "Job #{job.short_path} done"
+            end
+          end
+        end
+      end
+    end
+
+    app.register_common_parameter(:task_name, :symbol)
+  end
+end
+

data/lib/scout-camp.rb

@@ -1,7 +1,7 @@
 require 'scout'
 require 'scout/path'
 require 'scout/resource'
-Path.add_path :scout_camp, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
+Path.add_path :scout_camp_lib, File.join(Path.caller_lib_dir(__FILE__), "{TOPLEVEL}/{SUBPATH}")
 require 'scout/terraform_dsl'
 require 'scout/offsite'
 require 'scout/aws/s3'

data/scout-camp.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-camp 0.1.13 ruby lib
+# stub: scout-camp 0.1.14 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-camp".freeze
-  s.version = "0.1.13".freeze
+  s.version = "0.1.14".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/scout-camp",
+    "doc/terraform.md",
     "lib/scout-camp.rb",
     "lib/scout/aws/s3.rb",
     "lib/scout/offsite.rb",
@@ -34,10 +35,32 @@ Gem::Specification.new do |s|
     "lib/scout/offsite/ssh.rb",
     "lib/scout/offsite/step.rb",
     "lib/scout/offsite/sync.rb",
+    "lib/scout/render.rb",
+    "lib/scout/render/engine.rb",
+    "lib/scout/render/helpers.rb",
+    "lib/scout/render/resource.rb",
+    "lib/scout/sinatra/auth.rb",
+    "lib/scout/sinatra/base.rb",
+    "lib/scout/sinatra/base/assets.rb",
+    "lib/scout/sinatra/base/favicon.rb",
+    "lib/scout/sinatra/base/headers.rb",
+    "lib/scout/sinatra/base/helpers.rb",
+    "lib/scout/sinatra/base/parameters.rb",
+    "lib/scout/sinatra/base/post_processing.rb",
+    "lib/scout/sinatra/base/session.rb",
+    "lib/scout/sinatra/entity.rb",
+    "lib/scout/sinatra/finder.rb",
+    "lib/scout/sinatra/fragment.rb",
+    "lib/scout/sinatra/htmx.rb",
+    "lib/scout/sinatra/knowledge_base.rb",
+    "lib/scout/sinatra/tool.rb",
+    "lib/scout/sinatra/workflow.rb",
     "lib/scout/terraform_dsl.rb",
     "lib/scout/terraform_dsl/deployment.rb",
     "lib/scout/terraform_dsl/util.rb",
     "scout-camp.gemspec",
+    "scout_commands/find",
+    "scout_commands/glob",
     "scout_commands/offsite",
     "scout_commands/sync",
     "scout_commands/terraform/add/lambda",
@@ -111,16 +134,29 @@ Gem::Specification.new do |s|
     "test/scout/offsite/test_ssh.rb",
     "test/scout/offsite/test_step.rb",
     "test/scout/offsite/test_sync.rb",
+    "test/scout/render/test_engine.rb",
+    "test/scout/render/test_resource.rb",
+    "test/scout/sinatra/base/test_headers.rb",
+    "test/scout/sinatra/base/test_parameters.rb",
+    "test/scout/sinatra/test_base.rb",
+    "test/scout/sinatra/test_entity.rb",
+    "test/scout/sinatra/test_render.rb",
+    "test/scout/sinatra/test_workflow.rb",
     "test/scout/test_terraform_dsl.rb",
     "test/test_helper.rb"
   ]
   s.homepage = "http://github.com/mikisvaz/scout-camp".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.6.7".freeze
+  s.rubygems_version = "3.7.0.dev".freeze
   s.summary = "Deploy you scouts".freeze
 
   s.specification_version = 4
 
   s.add_runtime_dependency(%q<scout-essentials>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<aws-sdk-s3>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<sinatra>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<omniauth>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<mimemagic>.freeze, [">= 0".freeze])
+  s.add_runtime_dependency(%q<omniauth-google-oauth2>.freeze, [">= 0".freeze])
 end
 

data/scout_commands/find

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+
+require 'scout-camp'
+
+$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Find files
+
+$ #{$0} [<options>] (<resource> <path>|<path>)
+
+-h--help Print this help
+-r--requires* Files to require; 'all' for all in Scout.etc.requires
+-lw--load_workflow* Files to require; 'all' for all in Scout.etc.requires
+-w--where* Where to look for the path
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+resource, path = ARGV
+path, resource = resource, nil if path.nil?
+
+raise MissingParameterException.new(:path) if path.nil?
+
+case options[:workflows]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.workflows.list.each do |workflow|
+    Workflow.require_workflow file
+  end if Scout.etc.workflows.exists?
+else
+  options[:workflows].split(/[ ,;|]/).each do |workflow|
+    Workflow.require_workflow workflow
+  end
+end
+
+case options[:requires]
+when nil, false, "false", "none"
+when "all"
+  Scout.etc.requires.list.each do |file|
+    require file
+  end if Scout.etc.requires.exists?
+else
+  options[:requires].split(/[ ,;|]/).each do |file|
+    require file
+  end
+end
+
+location = nil
+where = options[:where]
+all = options[:all]
+
+begin
+  resource = Workflow.require_workflow resource
+rescue
+  begin
+    resource = Kernel.const_get(resource)
+  rescue
+    raise "Resource not found: #{ resource }"
+  end
+end if resource
+
+path = (resource || Scout)[path.dup]
+
+if where.nil? || where == 'all' || path.path_maps.include?(where.to_sym)
+  location = path.find(where)
+
+  if Array === location
+    puts location * "\n"
+  else
+    location = nil if ! Open.exists?(location)
+    puts location
+  end
+else
+  puts SSHLine.command(where, $0, ARGV, options.merge("where" => :all, add_option_dashes: true))
+end