tpt-rails 1.0.0

data/lib/generators/tpt_rails/continuous_integration/templates/ci/test.config.xml.tt

@@ -0,0 +1,76 @@
+<?xml version='1.1' encoding='UTF-8'?><org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject plugin="workflow-multibranch@2.21">
+  <actions/>
+  <description></description>
+  <displayName><%= Tpt::Rails.app_name %>-test</displayName>
+  <properties>
+    <org.jenkinsci.plugins.workflow.libs.FolderLibraries plugin="workflow-cps-global-lib@2.13">
+      <libraries>
+        <org.jenkinsci.plugins.workflow.libs.LibraryConfiguration>
+          <name>jenkins-pipeline-library</name>
+          <retriever class="org.jenkinsci.plugins.workflow.libs.SCMSourceRetriever">
+            <scm class="jenkins.plugins.git.GitSCMSource" plugin="git@3.9.3">
+              <id>31643265-3634-6665-6262-343134326436</id>
+              <remote>git@github.com:TeachersPayTeachers/jenkins-pipeline-library.git</remote>
+              <credentialsId>jenkins-master-rsa-key</credentialsId>
+              <traits>
+                <jenkins.plugins.git.traits.BranchDiscoveryTrait/>
+              </traits>
+            </scm>
+          </retriever>
+          <implicit>false</implicit>
+          <allowVersionOverride>true</allowVersionOverride>
+          <includeInChangesets>true</includeInChangesets>
+        </org.jenkinsci.plugins.workflow.libs.LibraryConfiguration>
+      </libraries>
+    </org.jenkinsci.plugins.workflow.libs.FolderLibraries>
+    <org.jenkinsci.plugins.pipeline.modeldefinition.config.FolderConfig plugin="pipeline-model-definition@1.3.7">
+      <dockerLabel/>
+      <registry plugin="docker-commons@1.14"/>
+    </org.jenkinsci.plugins.pipeline.modeldefinition.config.FolderConfig>
+  </properties>
+  <folderViews class="jenkins.branch.MultiBranchProjectViewHolder" plugin="branch-api@2.4.0">
+    <owner class="org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject" reference="../.."/>
+  </folderViews>
+  <healthMetrics>
+    <com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric plugin="cloudbees-folder@6.8">
+      <nonRecursive>false</nonRecursive>
+    </com.cloudbees.hudson.plugins.folder.health.WorstChildHealthMetric>
+    <jenkins.branch.PrimaryBranchHealthMetric plugin="branch-api@2.4.0"/>
+  </healthMetrics>
+  <icon class="jenkins.branch.MetadataActionFolderIcon" plugin="branch-api@2.4.0">
+    <owner class="org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject" reference="../.."/>
+  </icon>
+  <orphanedItemStrategy class="com.cloudbees.hudson.plugins.folder.computed.DefaultOrphanedItemStrategy" plugin="cloudbees-folder@6.8">
+    <pruneDeadBranches>true</pruneDeadBranches>
+    <daysToKeep>-1</daysToKeep>
+    <numToKeep>5</numToKeep>
+  </orphanedItemStrategy>
+  <triggers/>
+  <disabled>false</disabled>
+  <sources class="jenkins.branch.MultiBranchProject$BranchSourceList" plugin="branch-api@2.4.0">
+    <data>
+      <jenkins.branch.BranchSource>
+        <source class="org.jenkinsci.plugins.github_branch_source.GitHubSCMSource" plugin="github-branch-source@2.4.5">
+          <id>65633436-6138-6163-6638-323663356636</id>
+          <credentialsId>tpt-github-source-credential</credentialsId>
+          <repoOwner>TeachersPayTeachers</repoOwner>
+          <repository><%= Tpt::Rails.app_name %></repository>
+          <traits>
+            <org.jenkinsci.plugins.github__branch__source.BranchDiscoveryTrait>
+              <strategyId>3</strategyId>
+            </org.jenkinsci.plugins.github__branch__source.BranchDiscoveryTrait>
+            <org.jenkinsci.plugins.github__branch__source.TagDiscoveryTrait/>
+          </traits>
+        </source>
+        <strategy class="jenkins.branch.DefaultBranchPropertyStrategy">
+          <properties class="empty-list"/>
+        </strategy>
+      </jenkins.branch.BranchSource>
+    </data>
+    <owner class="org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject" reference="../.."/>
+  </sources>
+  <factory class="org.jenkinsci.plugins.workflow.multibranch.WorkflowBranchProjectFactory">
+    <owner class="org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject" reference="../.."/>
+    <scriptPath>ci/test.Jenkinsfile</scriptPath>
+  </factory>
+</org.jenkinsci.plugins.workflow.multibranch.WorkflowMultiBranchProject>

data/lib/generators/tpt_rails/continuous_integration/templates/ci/tptcd.Jenkinsfile.tt

@@ -0,0 +1,7 @@
+@Library('tpt-pipeline-library') _
+
+def ci = new tpt.CI(this)
+ci.project = '<%= Tpt::Rails.app_name %>'
+ci.buildCredentials = [[ id: '<%= Tpt::Rails.app_name %>-rails-master-key', var: '<%= Tpt::Rails.app_name.underscore.upcase %>_RAILS_MASTER_KEY' ]]
+
+tptcdPipeline(ci: ci)

data/lib/generators/tpt_rails/continuous_integration/templates/ci/tptci.Jenkinsfile.tt

@@ -0,0 +1,7 @@
+@Library('tpt-pipeline-library') _
+
+def ci = new tpt.CI(this)
+ci.project = '<%= Tpt::Rails.app_name %>'
+ci.buildCredentials = [[ id: '<%= Tpt::Rails.app_name %>-rails-master-key', var: '<%= Tpt::Rails.app_name.underscore.upcase %>_RAILS_MASTER_KEY' ]]
+
+tptciPipeline(ci: ci, ode: '') // we don't need an ode, just test this service

data/lib/generators/tpt_rails/kubernetes/kubernetes_generator.rb

@@ -0,0 +1,156 @@
+class Tpt::Rails::KubernetesGenerator < ::Rails::Generators::Base
+  VALID_APP_NAME = /^([a-z0-9]-?)+[a-z0-9]$/
+  ODE_KUBE_CONTEXT = 'kubernetes-eks-poc'
+
+  desc 'Set up the app with kubernetes'
+
+  source_root File.expand_path('templates', __dir__)
+
+  def setup_kubernetes
+    if !valid_app_name?
+      say("ERROR: App name #{Tpt::Rails.app_name.inspect} will not be valid for DNS usage.")
+      abort
+    end
+
+    check_dependencies
+
+    secrets_name = "#{Tpt::Rails.app_name}-secrets"
+
+    tpt_yaml = generate_tpt_yaml
+    team_name = tpt_yaml.fetch('owner')
+    app_type = tpt_yaml.fetch('target')
+
+    tpt_config = get_tpt_config
+    tpt_user_alias = tpt_config.fetch('user.alias')
+
+    default_email = get_default_email_address
+    email = ask("What's the maintainer email address? (#{default_email})").strip
+    email = email.presence || default_email || 'FIXME'
+
+    template 'Dockerfile'
+    template 'entrypoint.sh'
+    template '.dockerignore'
+
+    directory(
+      'kubernetes',
+      team_name: team_name,
+      secrets_name: secrets_name,
+      maintainer_email: email,
+      app_type: app_type,
+    )
+
+    inside('kubernetes/helm') do
+      run('helm dependency update', abort_on_failure: true)
+    end
+
+    add_kube_secret(secrets_name)
+
+    say(<<~MESSAGE)
+
+      ********************************************************************************
+      * Success!  Your project has been configured for Kubernetes!
+      ********************************************************************************
+
+      !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ! Follow-up actions required:
+      !
+      ! 1. Review and commit these changes
+      ! 2. Push your changes up to a Github repository
+      ! 3. Run `tpt project check`
+      ! 4. Deploy to an ODE with `tpt project push`
+      ! 5. Give "Read-only" access to the Docker Hub "readonly" user:
+      !   - Go to https://hub.docker.com/repository/docker/teacherspayteachers/#{Tpt::Rails.app_name}/permissions
+      !   - Choose "readonly" in the "Team" dropdown box
+      !   - Choose "Read-only" in the "Permission" box
+      !   - Click the "+" button on the right-hand side
+      ! 6. Monitor the status of your ODE deploy with:
+      !      kubectl get pods --namespace #{tpt_user_alias} | grep #{Tpt::Rails.app_name}
+      ! 7. When your ODE is ready you can visit it here:
+      !      http://#{Tpt::Rails.app_name}--#{tpt_user_alias}.ode.tptpm.info/#{Tpt::Rails::Engine.instance.routes.url_helpers.health_check_path}
+      !
+      ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    MESSAGE
+  end
+
+  private
+
+  def check_dependencies
+    say('Checking dependencies…')
+    check_dependency('config/master.key', 'test -f config/master.key')
+    check_dependency('jq', 'which jq')
+    check_dependency('tpt network', 'nc -G1 -w1 chartmuseum.tptpm.info 443')
+    check_dependency('tpt-cli', 'tpt version')
+    check_dependency('tpt config', 'tpt config get') # fails if tpt config has not been initialized
+    check_dependency('docker', 'docker version') # fails if docker isn't running
+    check_dependency('kubectl', 'kubectl version') # fails if kubectl isn't set up
+    check_dependency('kubeval', 'which kubeval') # needed for `tpt project push`. See https://kubeval.instrumenta.dev/installation/
+    check_dependency('helm', 'helm search justtesting') # fails if helm isn't installed + initialized (~/.helm directory created)
+    check_dependency('helm tpt repo', "helm repo list | egrep '^tpt '") # fails if tpt helm repo isn't added
+  end
+
+  def check_dependency(name, command)
+    say("Checking for dependency: #{name}")
+    run(command, abort_on_failure: true, capture: true) # capture: true makes the output nicer
+  rescue Exception => e
+    say "Error! #{name} does not seem to be available. `#{command}` returned: #{e.inspect}"
+    abort
+  end
+
+  # Dasherize the app name for kube usage, and verify that it will be valid for DNS usage.
+  # Note: Rails itself also verifies that the name does not start with a number so that it is a valid
+  #       Ruby identifier.
+  def valid_app_name?
+    Tpt::Rails.app_name =~ VALID_APP_NAME
+  end
+
+  def generate_tpt_yaml
+    # TODO: Add an options to `tpt project init` that allows passing in these values
+    say(<<~TEXT)
+
+      !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+      ! IMPORTANT !
+      ! In the prompt below, please enter "#{Tpt::Rails.app_name}" for the "project name"
+      !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+    TEXT
+
+    run('tpt project init --force', abort_on_failure: true)
+
+    YAML.load(run('cat tpt.yaml', capture: true, abort_on_failure: true))
+  end
+
+  def get_tpt_config
+    conf = run('tpt config get', capture: true, abort_on_failure: true)
+    YAML.load(conf)
+  end
+
+  def get_default_email_address
+    run('git config user.email', capture: true, abort_on_failure: false)&.strip.presence
+  end
+
+  def add_kube_secret(secrets_name)
+    secret = run('cat config/master.key', capture: true, abort_on_failure: true).strip
+    key_exists = run("kubectl get secret #{secrets_name} --context='#{ODE_KUBE_CONTEXT}' >/dev/null 2>&1", abort_on_failure: false)
+
+    if key_exists
+      say "Kube secret #{secrets_name} already exists. Updating it…"
+      ENV['KUBE_GENERATOR_RAILS_MASTER_KEY'] = Base64.strict_encode64(secret)
+      run(
+        <<~CMD,
+          kubectl get secret #{secrets_name} --context=#{ODE_KUBE_CONTEXT} --output=json \
+          | jq --arg key $KUBE_GENERATOR_RAILS_MASTER_KEY '.data["rails-master-key"]=$key' \
+          | kubectl apply --context=#{ODE_KUBE_CONTEXT} -f -
+        CMD
+        abort_on_failure: true,
+      )
+    else
+      say "Kube secret #{secrets_name} not found. Creating it…"
+      run(
+        "kubectl create secret generic #{secrets_name} --from-literal='rails-master-key=#{secret}' --context='#{ODE_KUBE_CONTEXT}'",
+        abort_on_failure: true,
+      )
+    end
+  end
+
+end

data/lib/generators/tpt_rails/kubernetes/templates/Dockerfile.tt

@@ -0,0 +1,41 @@
+FROM ruby:<%= RUBY_VERSION %>
+
+RUN mkdir /app
+WORKDIR /app
+
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash \
+  && apt-get update && apt-get install -y nodejs && rm -rf /var/lib/apt/lists/* \
+  && curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
+  && echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
+  && apt-get update && apt-get install -y yarn && rm -rf /var/lib/apt/lists/*
+
+COPY package.json yarn.lock /app/
+RUN yarn install --check-files
+
+# Set production env for asset compilation
+ARG RAILS_ENV=production
+ARG APP_ENV=production
+ARG APP_NAME=<%= Tpt::Rails.app_name %>
+
+ARG <%= Tpt::Rails.app_name.underscore.upcase %>_RAILS_MASTER_KEY
+# GITHUB_TOKEN allows accessing our private tpt-rails gem on Github
+ARG GITHUB_TOKEN
+
+COPY Gemfile /app/Gemfile
+COPY Gemfile.lock /app/Gemfile.lock
+RUN gem update bundler
+RUN BUNDLE_GITHUB__COM="${GITHUB_TOKEN}:x-oauth-basic" bundle install --without development test
+
+# Add a script to be executed every time the container starts.
+COPY entrypoint.sh /usr/bin/
+RUN chmod +x /usr/bin/entrypoint.sh
+
+ADD . /app
+
+RUN RAILS_MASTER_KEY=${<%= Tpt::Rails.app_name.underscore.upcase %>_RAILS_MASTER_KEY} bin/rails assets:precompile
+
+EXPOSE 3000
+
+# TODO: Delete this?
+ENTRYPOINT ["entrypoint.sh"]
+CMD ["rails", "server", "-b", "0.0.0.0"]

data/lib/generators/tpt_rails/kubernetes/templates/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+if [ -f /app/tmp/pids/server.pid ]; then
+  rm /app/tmp/pids/server.pid
+fi
+
+echo "Preparing database"
+# In Kubernetes this should always be a no-op because our migration hook will already have run.
+rails db:prepare
+
+echo "Starting server"
+exec "$@"

data/lib/generators/tpt_rails/kubernetes/templates/kubernetes/helm/Chart.yaml.tt

@@ -0,0 +1,16 @@
+# This is the entrypoint to the Helm Chart
+# Most properties here are pretty self explanatory
+
+name: <%= Tpt::Rails.app_name %>
+# Chart version - increase this whenever you change anything in the helm/ folder. tpt-cli will warn
+# you if you don't.
+version: 0.0.1
+# Application version - can be useful if we run multiple versions of the app at the same time.
+appVersion: 0.0.1
+description: <%= Tpt::Rails.app_name %> service
+keywords: [odev2, rails, service-template, '<%= Tpt::Rails.app_name %>', '<%= config.fetch(:app_type) %>']
+home: https://www.teacherspayteachers.com
+icon: https://static1.teacherspayteachers.com/tpt-frontend/releases/production/current/a24f03b12028dbc93cf182318e6993e5.svg
+sources: ['https://github.com/TeachersPayTeachers/<%= Tpt::Rails.app_name  %>']
+maintainers: [{name: '<%= config.fetch(:team_name) %>', email: '<%= config.fetch(:maintainer_email) %>'}]
+engine: gotpl # yaml parsing engine, just leave this alone

data/lib/generators/tpt_rails/kubernetes/templates/kubernetes/helm/README.md

@@ -0,0 +1,142 @@
+# Example Helm Chart
+
+This is an example [Helm Chart](https://helm.sh/docs/developing_charts/).
+Helm sits a layer above Kubernetes, and provides a simpler interface for defining Kubernetes entities across environments. It also provides a mechanism for declaring dependencies between Kubernetes services, although we're not currently using that feature. You can think of Helm as a package manager and deployment tool for Kubernetes entities.
+
+Currently, we use Helm to manage deployments and updates to our On Demand Environments (ODEs). Production deploys use Kubernetes directly, except for graphapi which is also using Helm.
+
+If you'd like your service to deploy to an ODE (you should!), you'll need to build a Chart for it and reference that chart from our [Charts repo](https://github.com/TeachersPayTeachers/charts). See [this branch](https://github.com/TeachersPayTeachers/charts/compare/adding-service-example) for a reference on how to do this.
+
+## General Chart Overview
+
+There are a couple files that constitute a Chart. Here is an overview:
+
+```
+(Project Root)
+└── kubernetes
+    └── helm
+        ├── charts
+        │   ├── copy-secrets-1.1.5.tgz
+        ├── templates
+        │   ├── _helpers.tpl
+        │   ├── autoscaler.yaml
+        │   ├── deployment.yaml
+        │   ├── NOTES.txt
+        │   └── service.yaml
+        ├── .helmignore
+        ├── Chart.yaml
+        ├── README.md
+        ├── requirements.lock
+        ├── requirements.yaml
+        ├── values.dev.yaml
+        ├── values.staging.yaml
+        └── values.yaml
+```
+
+### Chart.yaml
+
+This is Helm's entrypoint to the Chart. Here the high level metadata is specified, including the Chart name and description.
+
+### values.yaml
+
+Here you can configure any variables that your Chart will depend on. This includes properties like application secrets, and endpoint configuration of service dependencies. These variables can then be referenced in the Chart definition, to construct your Kubernetes entity.
+
+Usually, you want different configurations for different deployment environments. We use `values.yaml` to configure a standard production deployment, and then use `values.{env}.yaml` to override values where a given environment differs from production. In this service, we have created `values.dev.yaml` for configuring ODE deployments and `values.staging.yaml` to configure deployments to staging.
+
+### Secrets
+
+See [this wiki page](https://teacherspayteachers.atlassian.net/wiki/spaces/ENGINEERING/pages/64946180/Using+Secrets+in+Kubernetes) for information on how to define a secret in Kubernetes. After adding the secret to the cluster, you can reference it in a `values` file which will cause it to be passed to your application as an environmental variable:
+
+```yaml
+secrets:
+  - name: BUGSNAG_API_KEY
+    valueFrom:
+      secretKeyRef:
+        name: service-template-node-bugsnag
+        key: apiKey
+```
+
+If you wish to use a secret in ODEs environnments as well, will need to use the `copy-secrets` dependency chart so that the secrets are copied from the default namespace when a user spins up an ODE. See the [documentation for the chart](https://github.com/TeachersPayTeachers/charts/tree/master/copy-secrets) for more information. A service which needed a secret called `super-secret` when running in ODEs would have something like this in its `values.dev.yaml` file:
+
+```yaml
+secrets:
+  - name: SUPER_SECRET
+    valueFrom:
+      secretKeyRef:
+        name: super-secret
+        key: key
+
+copy-secrets:
+  secrets:
+    - super-secret
+```
+
+### /templates
+
+Helm will process all files in this directory, and combine them into a Kubernetes entity definition. These files might look like the Kube deployment specifications you're used to (e.g. for [tpt-frontend](https://github.com/TeachersPayTeachers/tpt-frontend/blob/master/kubernetes/deployment.production.yaml)), with one major difference: they can reference variables. These variables can come from a couple places, described in more detail below.
+
+### requirements.yaml
+
+This optional file lists dependencies for your chart. After adding or changing an entry in this file, you can download all dependencies into the `charts` folder and update `requirements.lock` with the command `helm dependency update`. Right now, the only dependency included is the `copy-secrets` chart mentioned above.
+
+## Template file syntax
+
+The meat of the Helm Chart is the `/templates` directory, which contains all the resources needed for Helm to construct and deploy a Kubernetes resource. These files mostly look like Kubernetes entity definitions, but with reference to variables. You can find documentation on what's available [here](https://helm.sh/docs/chart_template_guide/#built-in-objects), but below are couple common patterns you'll see.
+
+### myProperty: {{ .Values.foo.bar }}
+
+Inserts `someValue` as the value of myProperty, where `someValue` was defined in your `values.yaml` file:
+
+```yaml
+foo:
+  bar: someValue
+```
+
+### myProperty: {{ template "foo.bar" }}
+
+References a property that was specified by `_helpers.tpl`
+
+```
+{{- define "foo.bar" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride -}}
+{{- else -}}
+{{- .Values.nameOverride -}}
+{{- end -}}
+{{- end -}}
+```
+
+This is a good way of bringing in variables that need a bit of processing to transform them from raw `values.yaml` properties.
+
+### myProperty: "{{ .Release.Name }}"
+
+References Helm defined variables
+
+## Testing Your Helm Charts
+
+If this prints the contents of your helm charts without erroring, there are no syntax errors.
+
+`helm install --dry-run --debug </path/to/helm/chart/dir>`
+
+The TpT-CLI tool should be used to ensure your service will work with TpT ODEs v2.
+
+[TpT-CLI Github Repo](https://github.com/TeachersPayTeachers/tpt-cli)
+
+[ODE v2 Reference](https://teacherspayteachers.atlassian.net/wiki/spaces/ODE/pages/586678286/IN+PROGRESS+Getting+Started+with+ODE+Evolution+v+2.0+-+TpT-CLI)
+
+## Testing Your Service
+### Contract and API Functional Testing (IN PROGRESS)
+Helm test charts can be used to test your service in isolation. If you have tests defined in Postman (SUPPORTED) or your service repo (IN PROGRESS), you can execute them using the TpT-CLI test command (IN PROGRESS).
+
+**_Do not use `helm test` directly to run tests. This requires some extra coordination to not leave test pods running and still have access to test results. TpT-CLI will be updated with a tet command to handle this orchestration._**
+
+#### Usage
+1) Create a helm test chart under [templates/tests](templates/tests). See [test.apiFunctional.postman.yaml](templates/tests/test.apiFunctional.postman.yaml)
+2) Add any test configuration to the appropriate values.yaml.
+3) Verify your changes using `tpt project check`.
+4) Commit your changes.
+5) Release your charts using `tpt project push`.
+6) (IN PROGRESS) Execute the tests via the TPT-CLI test command.
+
+### Example Postman Tests
+https://tptapis.postman.co/collections/5817358-93d711ed-3181-480d-bba5-ab68b0a29eae?version=latest&workspace=d9a6b5ae-1a22-4d0b-b706-2cc818031f79