@govuk-pay/cli 0.0.16 → 0.0.18

package/resources/legacy-ruby-cli/lib/pay_cli/commands/local/files/docker-compose.erb

@@ -1,9 +1,7 @@
-version: '2.1'
-
 services:
   <% @dbs.each do |db| %>
     <%= db[:name] %>:
-      image: postgres:15.2
+      image: postgres:15.6
       environment:
         - POSTGRES_PASSWORD=mysecretpassword
       mem_limit: 250M
@@ -11,7 +9,7 @@ services:
         driver: "json-file"
       container_name: <%= db[:name] %>
       healthcheck:
-        test: ["CMD-SHELL", "pg_isready"]
+        test: ["CMD-SHELL", "pg_isready -d postgres -U postgres"]
         interval: 10s
         timeout: 5s
         retries: 5
@@ -19,6 +17,8 @@ services:
         - ./postgres/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
       ports:
         - "<%= db[:port] %>:5432"
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% @remote_java_apps.each do |app| %>
     <%= app[:name] %>:
@@ -48,6 +48,23 @@ services:
         - AWS_SECRET_ACCESS_KEY=mockSecretAccessKey
         <% end %>
 
+        <% if @egress_proxy_required && Config.can_use_egress_proxy(app[:name]) %>
+        - http_proxy=http://egress:8080
+        - HTTP_PROXY=http://egress:8080
+        - https_proxy=http://egress:8080
+        - HTTPS_PROXY=http://egress:8080
+        - HTTP_PROXY_HOST=egress
+        - HTTPS_PROXY_HOST=egress
+        - HTTP_PROXY_URI=http://egress:8080
+        - HTTPS_PROXY_URI=http://egress:8080
+        - HTTP_PROXY_ENABLED=true
+        - HTTPS_PROXY_ENABLED=true
+        - HTTP_PROXY_PORT=8080
+        - HTTPS_PROXY_PORT=8080
+        - HTTP_PROXY_SCHEME=http
+        - NO_PROXY=<%= @no_proxy_env_var %>
+        <% end %>
+
         <% if app[:queues] %>
         - AWS_SQS_REGION=eu-west-1
         - AWS_SQS_MESSAGE_MAXIMUM_WAIT_TIME_IN_SECONDS=20
@@ -83,6 +100,8 @@ services:
         - "<%= app[:port] %>:<%= app[:port] %>"
         <% if app[:admin_port] %>- "<%= app[:admin_port] %>:<%= app[:admin_port] %>"<% end %>
       container_name: <%= app[:name] %>
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% @local_java_apps.each do |app| %>
     <%= app[:name] %>:
@@ -112,6 +131,23 @@ services:
         - AWS_SECRET_ACCESS_KEY=mockSecretAccessKey
         <% end %>
 
+        <% if @egress_proxy_required && Config.can_use_egress_proxy(app[:name]) %>
+        - http_proxy=http://egress:8080
+        - HTTP_PROXY=http://egress:8080
+        - https_proxy=http://egress:8080
+        - HTTPS_PROXY=http://egress:8080
+        - HTTP_PROXY_HOST=egress
+        - HTTPS_PROXY_HOST=egress
+        - HTTP_PROXY_URI=http://egress:8080
+        - HTTPS_PROXY_URI=http://egress:8080
+        - HTTP_PROXY_ENABLED=true
+        - HTTPS_PROXY_ENABLED=true
+        - HTTP_PROXY_PORT=8080
+        - HTTPS_PROXY_PORT=8080
+        - HTTP_PROXY_SCHEME=http
+        - NO_PROXY=<%= @no_proxy_env_var %>
+        <% end %>
+
         <% if app[:queues] %>
         - AWS_SQS_REGION=eu-west-1
         - AWS_SQS_MESSAGE_MAXIMUM_WAIT_TIME_IN_SECONDS=20
@@ -149,6 +185,8 @@ services:
           - "<%= app[:port] %>:<%= app[:port] %>"
           <% if app[:admin_port] %>- "<%= app[:admin_port] %>:<%= app[:admin_port] %>"<% end %>
       container_name: <%= app[:name] %>
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% @remote_node_apps.each do |app| %>
     <%= app[:name] %>:
@@ -161,10 +199,25 @@ services:
       env_file: services/<%= app[:name] %>.env
       environment:
         - SECURE_COOKIE_OFF=true
-        - NODE_ENV=${NODE_ENV:-production}
         - RUN_APP=true
         - DISABLE_INTERNAL_HTTPS=true
         - PORT=<%= app[:port] %>
+        <% if @egress_proxy_required && Config.can_use_egress_proxy(app[:name]) %>
+        - http_proxy=http://egress:8080
+        - HTTP_PROXY=http://egress:8080
+        - https_proxy=http://egress:8080
+        - HTTPS_PROXY=http://egress:8080
+        - HTTP_PROXY_HOST=egress
+        - HTTPS_PROXY_HOST=egress
+        - HTTP_PROXY_URI=http://egress:8080
+        - HTTPS_PROXY_URI=http://egress:8080
+        - HTTP_PROXY_ENABLED=true
+        - HTTPS_PROXY_ENABLED=true
+        - HTTP_PROXY_PORT=8080
+        - HTTPS_PROXY_PORT=8080
+        - HTTP_PROXY_SCHEME=http
+        - NO_PROXY=<%= @no_proxy_env_var %>
+        <% end %>
         <%- @java_apps.each do |_app| -%>
         <%- if app[:name] == 'toolbox' && _app[:name] == 'publicauth' %>
         - PUBLIC_AUTH_URL=http://<%= _app[:name] %>:<%= _app[:port] %>
@@ -186,6 +239,8 @@ services:
       logging:
         driver: "json-file"
       container_name: <%= app[:name] %>
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% @local_node_apps.each do |app| %>
     <%= app[:name] %>:
@@ -196,12 +251,32 @@ services:
           condition: service_healthy
       <% end %>
       env_file: services/<%= app[:name] %>.env
+      <% if @mount_local_node_apps %>
+      volumes:
+        - "$WORKSPACE/pay-<%= app[:name] %>:/app"
+      <% end %>
       environment:
         - SECURE_COOKIE_OFF=true
         - RUN_APP=true
         - DISABLE_APPMETRICS=true
         - DISABLE_INTERNAL_HTTPS=true
         - PORT=<%= app[:port] %>
+        <% if @egress_proxy_required && Config.can_use_egress_proxy(app[:name]) %>
+        - http_proxy=http://egress:8080
+        - HTTP_PROXY=http://egress:8080
+        - https_proxy=http://egress:8080
+        - HTTPS_PROXY=http://egress:8080
+        - HTTP_PROXY_HOST=egress
+        - HTTPS_PROXY_HOST=egress
+        - HTTP_PROXY_URI=http://egress:8080
+        - HTTPS_PROXY_URI=http://egress:8080
+        - HTTP_PROXY_ENABLED=true
+        - HTTPS_PROXY_ENABLED=true
+        - HTTP_PROXY_PORT=8080
+        - HTTPS_PROXY_PORT=8080
+        - HTTP_PROXY_SCHEME=http
+        - NO_PROXY=<%= @no_proxy_env_var %>
+        <% end %>
         <%- @java_apps.each do |_app| -%>
         <%- if app[:name] == 'toolbox' && _app[:name] == 'publicauth' %>
         - PUBLIC_AUTH_URL=http://<%= _app[:name] %>:<%= _app[:port] %>
@@ -224,6 +299,8 @@ services:
       logging:
         driver: "json-file"
       container_name: <%= app[:name] %>
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% @proxies.each do |app| %>
     <%= app[:name] %>:
@@ -248,6 +325,8 @@ services:
       logging:
         driver: "json-file"
       container_name: <%= app[:name] %>
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% if @queue_apps.any? %>
     localstack:
@@ -260,6 +339,8 @@ services:
         - "4566:4566" # All AWS services exposed on this port
       volumes:
         - <%= @localstack_init_file %>:/etc/localstack/init/ready.d/init-aws.sh
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
   <% if @java_apps.any? {|app| Config.uses_redis(app[:name])} %>
     redis:
@@ -267,4 +348,28 @@ services:
       container_name: localRedis
       ports:
         - "6379:6379"
+      networks:
+        - pay_local_mimic_aws_vpc
   <% end %>
+  <% if @egress_proxy_required %>
+    egress:
+      image: governmentdigitalservice/pay-egress:latest-master
+      container_name: egress
+      volumes:
+        - "./services/egress/squid.conf:/etc/squid/squid.conf"
+      networks:
+        pay_local_mimic_aws_vpc:
+          ipv4_address: 172.18.0.253
+      expose:
+        - 8080
+  <% end %>
+
+networks:
+  pay_local_mimic_aws_vpc:
+    name: pay_local_mimic_aws_vpc
+    driver_opts:
+      com.docker.network.bridge.host_binding_ipv4: "127.0.0.1"
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.18.0.0/24

package/resources/legacy-ruby-cli/lib/pay_cli/commands/local/files/services/egress/squid.conf

@@ -0,0 +1,47 @@
+logformat govuk_datetime %tg %6tr %>a %Ss/%03>Hs %<st %rm %ru %[un %Sh/%<a %mt
+
+acl localnet src 10.0.0.0/8     # RFC1918 possible internal network
+acl localnet src 172.16.0.0/12  # RFC1918 possible internal network
+acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
+acl localnet src fc00::/7       # RFC4193 local private network range
+acl localnet src fe80::/10      # RFC4291 link-local (directly-plugged) machine
+
+acl SSL_ports port 443
+acl Safe_ports port 80
+acl purge method PURGE
+acl CONNECT method CONNECT
+
+acl all_sites                ssl::server_name_regex .*
+
+acl vpc src 172.18.0.0/16
+
+http_access allow all_sites        vpc SSL_ports
+
+http_access allow manager      localhost
+http_access deny  manager
+http_access allow purge        localhost
+http_access deny  purge
+http_access deny  !Safe_ports
+http_access deny  CONNECT      !SSL_ports
+http_access allow localhost
+
+http_access deny    all
+icp_access  deny    all
+
+http_port 0.0.0.0:8080
+
+access_log stdio:/dev/stdout govuk_datetime
+cache_log  stdio:/dev/stdout govuk_datetime
+
+refresh_pattern     ^ftp:     1440  20% 10080
+refresh_pattern     ^gopher:      1440  0%  1440
+refresh_pattern     -i (/cgi-bin/|\?)   0 0%  0
+refresh_pattern     (Release|Package(.gz)*)$  0 20% 2880
+
+pid_filename /squid/pid
+visible_hostname squid
+hosts_file /etc/hosts
+maximum_object_size 512 MB
+coredump_dir /squid
+cache_mem 1024 MB
+cache deny all

package/resources/legacy-ruby-cli/lib/pay_cli/commands/local.rb

@@ -14,17 +14,21 @@ require 'pathname'
 REQUIRES_COMPILATION = %w[frontend products-ui selfservice toolbox].freeze
 
 class PayCLI::Commands::Local < Thor
+  CLUSTERS=['admin', 'card', 'paymentlinks', 'webhooks', 'endtoend', 'java', 'toolbox']
+
   warn "🏠  Local Pay for local people\n\n"
   map "up" => :launch
-  desc 'launch | up [--cluster <cluster>] [--local <app>] [--proxy] [--apps] [--rebuild]',
+  desc 'launch | up [--cluster <cluster>] [--local <app>] [--proxy] [--apps] [--rebuild] [--with-egress-proxy]',
        'Launch a predefined cluster, or a user specified list of apps.' \
        ' Specify a list of apps that should use your locally checked out code using --local'
   option :local, :type => :array, :default => []
   option :proxy, :type => :boolean, :default => false
-  option :cluster, :enum => ['admin', 'card', 'paymentlinks', 'webhooks', 'endtoend', 'java', 'toolbox']
+  option :cluster, :enum => CLUSTERS
   option :apps, :type => :array
   option :rebuild, :type => :boolean, :default => true
   option :experimental, :type => :boolean, :default => false
+  option :with_egress_proxy, :type => :boolean, :default => false
+  option :mount_local_node_apps, :type => :boolean, :default => false, :desc => "Mount your apps directory in your \$WORKSPACE folder into the container, this makes local changes take effect immedaitely in the container via nodemon"
   option :pull, :type => :boolean, :default => true, :desc => "Pull the latest versions of containers prior to launching, override with --no-pull"
 
   def launch()
@@ -42,6 +46,8 @@ class PayCLI::Commands::Local < Thor
       apps = all
     end
 
+    @egress_proxy_required = options[:with_egress_proxy]
+
     @node_apps = Config::node apps
     @java_apps = Config::java apps
 
@@ -51,6 +57,8 @@ class PayCLI::Commands::Local < Thor
     @remote_node_apps = Config.remote local_app_names, @node_apps
     @remote_java_apps = Config.remote local_app_names, @java_apps
 
+    @mount_local_node_apps = options[:mount_local_node_apps]
+
     @dbs = Config.db_configs Config.has_db apps
     @queue_apps = Config.has_queue apps
     @sns_topic_apps = Config.has_sns_topics apps
@@ -60,6 +68,17 @@ class PayCLI::Commands::Local < Thor
 
     @proxies = if options[:proxy] then Config::proxy_configs Config::has_proxy apps else [] end
 
+    @no_proxy_env_var = [
+      "localhost",
+      "127.0.0.1",
+      "172.18.0.253",
+      "localstack",
+      "redis",
+      Config.all.map { |app| app[:name] },
+      @dbs.map { |app| app[:name] },
+      @proxies.map { |app| app[:name] },
+    ].flatten!.join(",")
+
     @localstack_init_file = File.realpath(
       File.join(__dir__, 'local', 'files', 'localstack', 'init-aws.sh')
     )
@@ -106,13 +125,19 @@ class PayCLI::Commands::Local < Thor
     AppClient::report_state apps
   end
 
-  desc 'down --cluster <cluster>',
+  desc 'down [--cluster <cluster>]',
        'Bring down a cluster'
-  option :cluster, :enum => ['admin', 'card', 'paymentlinks', 'webhooks', 'endtoend']
+  option :cluster, :enum => CLUSTERS
   def down()
-    cluster = options[:cluster]
+    cluster = options.fetch(:cluster, 'all')
+    unless Docker::compose_file_exists_for_cluster?(cluster)
+      $stderr.puts "Error: cluster #{cluster} has not been launched, its docker-compose file was not found"
+      return
+    end
+
     warn "🛬  bringing down #{cluster} cluster"
     puts Docker::down cluster
+    puts Docker::cleanup_old_network
   end
 
   desc 'healthcheck <cluster>',
@@ -136,15 +161,21 @@ class PayCLI::Commands::Local < Thor
     end
     Docker::remove('localstack')
     Docker::remove('localRedis')
+    Docker::remove('egress')
+    Docker::remove_network
   end
 
-  desc 'restart',
+  desc 'restart [--cluster <cluster>] <app>',
        'Restart a container and waits for it to be healthy.'
-
+  option :cluster, :enum => CLUSTERS
   def restart(app_name)
-    warn "😅  restarting #{app_name}"
+    cluster = options.fetch(:cluster, 'all')
+    unless Docker::compose_file_exists_for_cluster?(cluster)
+      $stderr.puts "error: cluster #{cluster} has not been launched, its docker-compose file was not found"
+      return
+    end
 
-    puts `docker restart #{app_name}`
+    Docker::restart(cluster, app_name)
     app = Config::app_by_name(app_name)
     AppClient::wait_for_apps(app)
     AppClient::report_state(app)

package/resources/legacy-ruby-cli/lib/pay_cli/commands/ssm.rb

@@ -6,11 +6,10 @@ module PayCLI::Commands::Ssm
 
   def self.usage!
     STDERR.puts <<~USAGE
-      pay ssm ...
+      pay ssm <environment> <instance_id>
 
-      - `pay ssm ci-5 build` - connect to build (jenkins) in ci-5
-      - `pay ssm test-12 i-123456` - connect to instance i-123456 in test-12
-      - `pay ssm staging-2 asg toolbox` - connect to a box in the toolbox asg
+      example:
+        pay ssm test-12 i-1c472d8c2ffd826d7
 
     USAGE
     exit
@@ -18,36 +17,15 @@ module PayCLI::Commands::Ssm
 
   def self.start!
     args = ARGV[1..-1] || []
-    usage! if args.empty?
+    usage! if args.length != 2
 
     PayCLI::StopYubicoAuthenticator.stop_yubico_authenticator!
 
-    env, *args = args
-    usage! if args.length < 1
-    PayCLI::Environment.setup! env
-
-    resource, resource_name, *args = args
-    server_roles = ['build', 'deploy', 'admin', 'bastion']
-
-    instance_id = case
-    when resource.match(/^i-/)
-      resource
-    when server_roles.include?(resource)
-      get_instance_id env, resource
-    when resource == 'asg'
-      get_asg_instance_id env, resource_name
-    when resource == 'app'
-      warn <<~WARN
-
-        `pay ssm app ...` no longer works since the move to fargate.
+    env, instance_id = args
 
-        Please see usage below for other examples
+    usage! unless instance_id.match(/^i-/)
 
-      WARN
-      usage!
-    else
-      abort "Unknown resource type #{resource}"
-    end
+    PayCLI::Environment.setup! env
 
     warn <<~WARN
       \e[33m
@@ -67,37 +45,6 @@ module PayCLI::Commands::Ssm
     start_ssm_session(instance_id, env)
   end
 
-  def self.get_instance_id(env, resource)
-    ec2_client = Aws::EC2::Client.new
-    instances = ec2_client.describe_instances({
-      filters: [
-        {name: 'tag:Environment', values: [env]},
-        {name: 'tag:Role', values: [resource == 'admin' ? 'bastion' : resource]}
-      ]
-    }).reservations.first.instances
-
-    abort "No healthy admin instances found" if instances.empty?
-
-    instances.first.instance_id
-  end
-
-  def self.get_asg_instance_id(env, service)
-    STDERR.puts "Getting instance_id from asg #{service} in #{env}"
-
-    asg_name   = PayCLI::Naming.asg_name(env, service)
-    asg_client = Aws::AutoScaling::Client.new
-
-    asgs       = asg_client.describe_auto_scaling_groups({
-      auto_scaling_group_names: [asg_name],
-    }).auto_scaling_groups
-    abort "No ASGs found for #{service} -> #{asg_name}" if asgs.empty?
-
-    instances = asgs.first.instances
-    abort "No instances found for #{service} -> #{asg_name}" if instances.empty?
-
-    instances.first.instance_id
-  end
-
   def self.start_ssm_session(instance_id, env)
     STDERR.puts "SSM to instance #{instance_id} in #{env}"
 

package/resources/legacy-ruby-cli/lib/pay_cli/entry_point.rb

@@ -15,18 +15,9 @@ class PayCLI::EntryPoint < Thor
     PayCLI::Commands::Doctor.new.start!
   end
 
-  desc 'deployment_status <env>', 'Describe whats deployed'
-  def deployment_status(env, regex = '')
-    PayCLI::Commands::Healthcheck.deployment_status!(env)
-  end
-
-  desc 'scale', 'Quick way to scale microservices (dangerous)'
-
   desc 'secrets', 'Manage secrets in and between environments'
   subcommand 'secrets', PayCLI::Commands::Secrets
 
-  desc 'ssh', 'DEPRECATED: SSH to boxes in environments'
-
   desc 'ssm', 'Start an SSM session to boxes in environments'
 
   def ssm(*_args)
@@ -39,8 +30,6 @@ class PayCLI::EntryPoint < Thor
     PayCLI::Commands::Tf.start!
   end
 
-  desc 'tunnel', 'DEPRECATED: Opens tunnels to microservice(s).'
-
   desc 'local', 'Sets up local Pay development environment'
   subcommand 'local', PayCLI::Commands::Local
 

package/resources/legacy-ruby-cli/lib/pay_cli/environment.rb

@@ -10,7 +10,7 @@ module PayCLI::Environment
   end
 
   def self.setup_each!(env_filter, &block)
-    all_envs = %w{dev ci deploy test staging production}
+    all_envs = %w{dev deploy test staging production}
     if env_filter.any?
       envs = env_filter & all_envs
     else

package/resources/legacy-ruby-cli/lib/pay_cli/secrets.rb

@@ -118,7 +118,7 @@ module PayCLI::Secrets
     env['PASSWORD_STORE_DIR'] = path if path
 
     stdin, stdout, wait_thr = Open3.popen2(env, "pass #{pass_path}")
-    password = stdout.readline.chomp
+    password = stdout.readlines.map(&:chomp).join("\n")
     stdin.close
     stdout.close
     pass_status = wait_thr.value

package/resources/usageDetails.txt

@@ -8,4 +8,5 @@ Commands:
   pay schema                   # Generates web based database diagrams and me...
   pay secrets                  # Manage secrets in and between environments
   pay ssm                      # Start an SSM session to boxes in environments
+  pay tunnel                   # Open tunnel to application database
   pay tf                       # Runs Terraform