et_full_system 1.0.0.pre35 → 1.0.0.pre44

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b558d6da7aecc50f611c18ddbba78fecdaca265023cad027a7a99d45c098f765
-  data.tar.gz: 8a9ab67ad771f5aa423b92fe1f72ee6f19cdfc3945613ddcf9a030325735a63d
+  metadata.gz: cf35411960bdb0f229d00ca9f7b3643267748c79575f61c968504853b9e9a91e
+  data.tar.gz: 297eb0c8f4eff759d622c2056cf33c43a615ba5f706f402d80c143e12276ad47
 SHA512:
-  metadata.gz: e2e58153c14192b411d6df91b0fdde9528f7b2fc2314fe598ec4aec6807214f05711bb3189a141fe5d5ef9012e3aa351e199e4737f1cb0b036b4698c6c69b2a0
-  data.tar.gz: be1cab50995b442d34cfaee8dfcd6ac189451faa21bc8d18d60b8ce57681dc02c71a336109ae9482e95db0252a4cbfa3bdcc9240f5ab8090255bf7092e4b2308
+  metadata.gz: ab906c2b10b4aa36241ba1f77e2a01d926429f3b7b69cdcb55c48999bd93f0bb748bdc015d8c0ae51597027643ecf8e6a9c335420d3c41f09adb6d0bbdeaa200
+  data.tar.gz: bbbc699a6184b5750fef9ee326c7fd671bf0a55aa22473e3e7961d074ba54c4823bb675e767ce8c477c5f5b257087fcf6ac22a5ae0c816348a2e6f90923ff5da

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    et_full_system (1.0.0.pre34)
+    et_full_system (1.0.0.pre44)
       aws-sdk-s3 (~> 1.9)
       azure-storage (~> 0.15.0.preview)
       dotenv (~> 2.7, >= 2.7.2)
@@ -23,7 +23,7 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.260.0)
+    aws-partitions (1.261.0)
     aws-sdk-core (3.86.0)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
@@ -65,7 +65,7 @@ GEM
       rotp (~> 5.1)
       thor (~> 0.20.3)
       tilt (~> 2.0, >= 2.0.9)
-    faraday (0.17.1)
+    faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
@@ -86,13 +86,14 @@ GEM
     multi_json (1.14.1)
     multi_xml (0.6.0)
     multipart-post (2.1.1)
-    mustermann (1.0.3)
+    mustermann (1.1.1)
+      ruby2_keywords (~> 0.0.1)
     nokogiri (1.10.7)
       mini_portile2 (~> 2.4.0)
-    public_suffix (4.0.2)
+    public_suffix (4.0.3)
     puma (3.12.2)
     rack (2.0.8)
-    rack-protection (2.0.7)
+    rack-protection (2.0.8.1)
       rack
     rake (10.5.0)
     roda (3.27.0)
@@ -100,17 +101,18 @@ GEM
     rotp (5.1.0)
       addressable (~> 2.5)
     ruby-mcrypt (0.2.0)
-    sinatra (2.0.7)
+    ruby2_keywords (0.0.1)
+    sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
-      rack-protection (= 2.0.7)
+      rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
-    sinatra-contrib (2.0.7)
+    sinatra-contrib (2.0.8.1)
       backports (>= 2.8.2)
       multi_json
       mustermann (~> 1.0)
-      rack-protection (= 2.0.7)
-      sinatra (= 2.0.7)
+      rack-protection (= 2.0.8.1)
+      sinatra (= 2.0.8.1)
       tilt (~> 2.0)
     thor (0.20.3)
     thread_safe (0.3.6)

data/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   et:
-    image: 'et_full_system:${GEM_VERSION:-unknown}'
+    image: 'et_full_system:1.0.0.pre'
     build:
       context: ../
       dockerfile: docker/Dockerfile
@@ -12,6 +12,8 @@ services:
       - '${SERVER_PORT:-3100}:${SERVER_PORT:-3100}'
       - '${TRAEFIX_PORT:-3200}:${TRAEFIK_PORT:-3200}'
       - '${SMTP_PORT:-1025}:${SMTP_PORT:-1025}'
+    extra_hosts:
+      - "localhost.from.docker:${LOCALHOST_FROM_DOCKER_IP}"
     volumes:
       - rvm:/usr/local/rvm
       - global_node_modules:/usr/lib/node_modules
@@ -90,6 +92,81 @@ services:
       - et_full_system
     volumes:
       - redis_data:/data
+  selenium-hub:
+    image: selenium/hub:3.141.59-yttrium
+    container_name: selenium-hub
+    ports:
+      - "4444:4444"
+    networks:
+      - et_full_system
+    extra_hosts:
+      - "localhost.from.docker:${LOCALHOST_FROM_DOCKER_IP}"
+  chrome:
+    image: selenium/node-chrome:3.141.59-yttrium
+    volumes:
+      - /dev/shm:/dev/shm
+    depends_on:
+      - selenium-hub
+    environment:
+      - HUB_HOST=selenium-hub
+      - HUB_PORT=4444
+    extra_hosts:
+      - "localhost.from.docker:${LOCALHOST_FROM_DOCKER_IP}"
+    networks:
+      - et_full_system
+  firefox:
+    image: selenium/node-firefox:3.141.59-yttrium
+    volumes:
+      - /dev/shm:/dev/shm
+    depends_on:
+      - selenium-hub
+    environment:
+      - HUB_HOST=selenium-hub
+      - HUB_PORT=4444
+    extra_hosts:
+      - "localhost.from.docker:${LOCALHOST_FROM_DOCKER_IP:-127.255.255.255}"
+    networks:
+      - et_full_system
+  zalenium:
+    image: "dosel/zalenium"
+    container_name: zalenium
+    hostname: zalenium
+    tty: true
+    volumes:
+      - zalenium_videos:/home/seluser/videos
+      - /var/run/docker.sock:/var/run/docker.sock
+    ports:
+      - 4444:4444
+    extra_hosts:
+      - "localhost.from.docker:${LOCALHOST_FROM_DOCKER_IP}"
+    networks:
+      - et_full_system
+    command: >
+      start --desiredContainers 2
+            --maxDockerSeleniumContainers 8
+            --screenWidth 1024 --screenHeight 768
+            --timeZone "Europe/Berlin"
+            --videoRecordingEnabled false
+            --sauceLabsEnabled false
+            --browserStackEnabled false
+            --testingBotEnabled false
+            --cbtEnabled false
+            --lambdaTestEnabled false
+            --startTunnel false
+    environment:
+      - HOST_UID
+      - HOST_GID
+      - SAUCE_USERNAME
+      - SAUCE_ACCESS_KEY
+      - BROWSER_STACK_USER
+      - BROWSER_STACK_KEY
+      - TESTINGBOT_KEY
+      - TESTINGBOT_SECRET
+      - CBT_USERNAME
+      - CBT_AUTHKEY
+      - LT_USERNAME
+      - LT_ACCESS_KEY
+      - PULL_SELENIUM_IMAGE=true
 volumes:
   rvm:
   global_node_modules:
@@ -110,6 +187,7 @@ volumes:
   atos_bundle:
   db_data:
   redis_data:
+  zalenium_videos:
 
 networks:
   et_full_system:

data/lib/et_full_system/cli/docker.rb

@@ -22,7 +22,7 @@ module EtFullSystem
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
         cmd = "/bin/bash --login -c \"cd /home/app/full_system && et_full_system docker bootstrap && et_full_system local setup\""
-        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml run --rm et #{cmd}"
+        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml run --rm et #{cmd}"
         puts compose_cmd
         exec(compose_cmd)
       end
@@ -32,19 +32,20 @@ module EtFullSystem
     def compose(*args)
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
-        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml #{args.join(' ')}"
+        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml #{args.join(' ')}"
         puts cmd
         exec(cmd)
       end
     end
 
     desc "invoker", "Provides access to the invoker system running inside docker"
-    def invoker(*args)
+    def invoker(*args, show_output: true, show_command: true)
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
         cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et bash -lc \"invoker #{args.join(' ')}\""
-        puts cmd
-        exec(cmd)
+        puts cmd if show_command
+        result = `#{cmd}`
+        puts result if show_output
       end
     end
 
@@ -52,10 +53,10 @@ module EtFullSystem
     def reset
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
-        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml down -v"
+        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml down -v"
         puts cmd
         next unless system(cmd)
-        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml build --no-cache"
+        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml build --no-cache"
         puts cmd
         next unless system(cmd)
         self.class.start(['setup'])
@@ -67,9 +68,9 @@ module EtFullSystem
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
         cmd = "/bin/bash --login -c \"et_full_system local update_service_url #{service} #{url}\""
-        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et #{cmd}"
+        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et #{cmd}"
         puts compose_cmd
-        exec(compose_cmd)
+        `#{compose_cmd}`
       end
     end
 
@@ -81,24 +82,102 @@ module EtFullSystem
     desc "local_et1 PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the web server and sidekiq locally"
     def local_et1(port)
       local_service('et1', port)
-      invoker :remove, :et1_web
-      invoker :remove, :et1_sidekiq
-      puts "ET1 is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker service_env et1 > .env.local'"
+      invoker 'remove', 'et1_web'
+      invoker 'remove', 'et1_sidekiq'
+      puts "ET1 is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker et1_env > .env.local'"
     end
 
     desc "reset_et1", "Configures the reverse proxy and invoker to use the internal systems instead of local"
     def reset_et1
-      invoker :add, :et1_web
-      invoker :add, :et1_sidekiq
+      invoker 'add', 'et1_web'
+      invoker 'add', 'et1_sidekiq'
       puts "ET1 is now being hosted from inside docker container"
     end
 
+    desc "et1_env", "Shows et1's environment variables as they should be on a developers machine running locally"
+    def et1_env
+      service_env('et1')
+    end
+
+    desc "local_ccd_export", "Disables the sidekiq process in the invoker system to allow a developer to run it locally"
+    def local_ccd_export
+      invoker 'remove', 'et_ccd_export_sidekiq'
+      puts "ccd_export is now expected to be running on your machine. To configure your environment, run 'et_full_system docker ccd_export_env > .env.local'"
+    end
+
+    desc "reset_ccd_export", "Configures invoker to use the internal systems instead of local"
+    def reset_ccd_export
+      invoker 'add', 'et_ccd_export_sidekiq'
+      puts "ccd_export is now being run from inside docker container"
+    end
+
+    desc "ccd_export_env", "Shows ccd_export's environment variables as they should be on a developers machine running locally"
+    def ccd_export_env
+      service_env('et_ccd_export')
+    end
+
+    desc "local_api PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the web server and sidekiq locally"
+    def local_api(port)
+      local_service('api', port)
+      invoker 'remove', 'api_web'
+      invoker 'remove', 'api_sidekiq'
+      puts "api is now expected to be hosted on port #{port} on your machine. Also, you must provide your own sidekiq. To configure your environment, run 'et_full_system docker api_env > .env.local'"
+    end
+
+    desc "reset_api", "Configures the reverse proxy and invoker to use the internal systems instead of local"
+    def reset_api
+      invoker 'add', 'api_web'
+      invoker 'add', 'api_sidekiq'
+      puts "api is now being hosted from inside docker container"
+    end
+
+    desc "api_env", "Shows api's environment variables as they should be on a developers machine running locally"
+    def api_env
+      service_env('api')
+    end
+
+    desc "local_admin PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the admin web server locally"
+    def local_admin(port)
+      local_service('admin', port)
+      invoker 'remove', 'admin_web'
+      puts "Admin is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker admin_env > .env.local'"
+    end
+
+    desc "reset_admin", "Configures the reverse proxy and invoker to use the internal systems instead of local"
+    def reset_admin
+      invoker 'add', 'admin_web'
+      puts "Admin is now being hosted from inside docker container"
+    end
+
+    desc "admin_env", "Shows admin's environment variables as they should be on a developers machine running locally"
+    def admin_env
+      service_env('admin')
+    end
+
+    desc "local_et3 PORT", "Configures the reverse proxy and the invoker system to allow a developer to run the et3 web server locally"
+    def local_et3(port)
+      local_service('et3', port)
+      invoker 'remove', 'et3_web'
+      puts "ET3 is now expected to be hosted on port #{port} on your machine. To configure your environment, run 'et_full_system docker et3_env > .env.local'"
+    end
+
+    desc "reset_et3", "Configures the reverse proxy and invoker to use the internal systems instead of local"
+    def reset_et3
+      invoker 'add', 'et3_web'
+      puts "ET3 is now being hosted from inside docker container"
+    end
+
+    desc "et3_env", "Shows et3's environment variables as they should be on a developers machine running locally"
+    def et3_env
+      service_env('et3')
+    end
+
     desc "service_env SERVICE", "Returns the environment variables configured for the specified service"
     def service_env(service)
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
         cmd = "/bin/bash --login -c \"et_full_system local service_env #{service}\""
-        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et #{cmd}"
+        compose_cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml exec et #{cmd}"
         result = `#{compose_cmd}`
         replace_db_host_port(result)
         replace_redis_host_port(result)
@@ -112,7 +191,7 @@ module EtFullSystem
     def run_compose_command(*args, silent: false)
       Bundler.with_original_env do
         gem_root = File.absolute_path('../../..', __dir__)
-        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml #{args.join(' ')}"
+        cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml #{args.join(' ')}"
         puts cmd unless silent
         `#{cmd}`
       end

data/lib/et_full_system/cli/docker/server.rb

@@ -10,11 +10,16 @@ module EtFullSystem
         method_option :without, type: :array, default: [], banner: "service1 service2", desc: "If specified, disables the specified services from running. The services are et1_web, et1_sidekiq, et3_web, mail_web, api_web, api_sidekiq, admin_web, atos_api_web, s3_web, azure_blob_web, fake_acas_web, fake_ccd_web"
         method_option :ccd_docker, type: :boolean, default: false, aliases: 'ccd-docker', desc: "If specified, instead of using the built in fake ccd server, the system will connect to your local machine (see ccd-docker-host option also)"
         method_option :ccd_docker_host, type: :string, default: 'docker.for.mac.localhost', aliases: 'ccd-docker-host', desc: "Only used if ccd-docker=true.  This specifies the host name of your machine when viewed from inside the docker container.  This defaults to docker.for.mac.localhost which is suitable for mac OSX only.  Consult docker documentation for other systems"
+        method_option :with_test, type: :boolean, default: false, aliases: 'with-test'
+        method_option :use_selenium, type: :boolean, default: true, aliases: 'use-selenium', desc: 'Only used if with_test is true - says to use selenium in preference to zalenium'
+        method_option :chrome_instances, type: :numeric, default: 1, aliases: 'chrome-instances', desc: 'Specify the number of chrome instances for selenium'
+        method_option :firefox_instances, type: :numeric, default: 1, aliases: 'firefox-instances', desc: 'Specify the number of firefox instances for selenium'
         def up(*args)
           Bundler.with_original_env do
             server_args = []
             server_args << "--without=#{options[:without].join(' ')}" unless options[:without].empty?
             env_vars = ["SERVER_ARGS='#{server_args.join(' ')}'"]
+            env_vars << "LOCALHOST_FROM_DOCKER_IP=#{host_ip}"
             if options.ccd_docker?
               env_vars << "CCD_AUTH_BASE_URL=http://#{options.ccd_docker_host}:4502"
               env_vars << "CCD_IDAM_BASE_URL=http://#{options.ccd_docker_host}:5000"
@@ -26,9 +31,18 @@ module EtFullSystem
               env_vars << "CCD_SIDAM_PASSWORD=Pa55word11"
               env_vars << "CCD_GENERATE_ETHOS_CASE_REFERENCE=true"
             end
-
+            extra_args = []
+            if options.with_test? && options.use_selenium?
+              extra_args.concat(["--scale chrome=#{options.chrome_instances}"])
+              extra_args.concat(["--scale firefox=#{options.firefox_instances}"])
+            else
+              extra_args.concat(['--scale selenium-hub=0', '--scale chrome=0', '--scale firefox=0'])
+            end
+            unless options.with_test? && !options.use_selenium?
+              extra_args.concat(['--scale zalenium=0'])
+            end
             gem_root = File.absolute_path('../../../..', __dir__)
-            cmd = "GEM_VERSION=#{EtFullSystem::VERSION} #{env_vars.join(' ')} docker-compose -f #{gem_root}/docker/docker-compose.yml up #{args.join(' ')}"
+            cmd = "GEM_VERSION=#{EtFullSystem::VERSION} #{env_vars.join(' ')} docker-compose -f #{gem_root}/docker/docker-compose.yml up #{(extra_args + args).join(' ')}"
             puts cmd
             exec(cmd)
           end
@@ -38,13 +52,20 @@ module EtFullSystem
         def down(*args)
           ::Bundler.with_original_env do
             gem_root = File.absolute_path('../../../..', __dir__)
-            cmd = "GEM_VERSION=#{EtFullSystem::VERSION} docker-compose -f #{gem_root}/docker/docker-compose.yml down #{args.join(' ')}"
+            cmd = "GEM_VERSION=#{EtFullSystem::VERSION} LOCALHOST_FROM_DOCKER_IP=#{host_ip} docker-compose -f #{gem_root}/docker/docker-compose.yml down #{args.join(' ')}"
             puts cmd
             exec(cmd)
           end
         end
 
         default_task :up
+
+        private
+
+        def host_ip
+          result = JSON.parse `docker network inspect \`docker network list | grep docker_et_full_system | awk '{print $1}'\``
+          result.first.dig('IPAM', 'Config').first['Gateway']
+        end
       end
     end
   end

data/lib/et_full_system/cli/local.rb

@@ -139,7 +139,7 @@ module EtFullSystem
         'api' => :et_api,
         'et1' => :et1,
         'et3' => :et3,
-        'et_ccd_export' => :et_ccd_export_sidekiq
+        'et_ccd_export' => :et_ccd_export
       }
       file = lookup.fetch(service)
       parsed = Dotenv.parse("#{GEM_PATH}/foreman/.env", "#{GEM_PATH}/foreman/#{file}.env")

data/lib/et_full_system/version.rb

@@ -1,3 +1,3 @@
 module EtFullSystem
-  VERSION = "1.0.0.pre35"
+  VERSION = "1.0.0.pre44"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: et_full_system
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre35
+  version: 1.0.0.pre44
 platform: ruby
 authors:
 - Gary Taylor
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-27 00:00:00.000000000 Z
+date: 2020-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor