@hiiretail/gcp-infra-cli 0.104.0 → 1.0.0

package/README.md

@@ -1,69 +1,32 @@
-![npm (scoped)](https://img.shields.io/npm/v/@hiiretail/gcp-infra-cli)
-[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=extenda_gcp-infra-cli&metric=alert_status&token=6d74e0813df558de4721f4651fed4bad251c99e8)](https://sonarcloud.io/dashboard?id=extenda_gcp-infra-cli)
-
 # gcp-infra-cli
 
-This project is the CLI used to generate Terragrunt specifications for GCP infrastructure.
+This package is the CLI used to generate Terragrunt specifications for GCP infrastructure.
 
 The CLI groups resource-specific generators into commands.
 
-  * `init` -- Initialize repositories for infrastructure
-  * `resources` -- Create GCP resources
+* `init` -- Initialize repositories for infrastructure
+* `clan-resources` -- Create GCP resources
+* `common-resources` -- Create GCP resources
+* `docs` - Create documentation templates
 
-The normal workflow is to run `init` once in a repository and then use the `resources` to
+The normal workflow is to run `init` once in a repository and then use one of the resources generators to
 add GCP resources required by the developed services.
 
 See the [`tf-infra-gcp` clan resources](https://github.com/extenda/tf-infra-gcp/tree/master/docs/clan-resources.md)
 documentation for a in-depth description of the contents this CLI is able to generate.
 
-
 ## Usage
 
 Download the CLI using NPM.
 
 ```bash
-$ npm install --global @hiiretail/gcp-infra-cli
+npm install --global @hiiretail/gcp-infra-cli
 ```
 
 If run without the arguments, the help will be displayed.
 
 ```
-$ gcp-infra --help
-```
-
-See detail documentation for all supported commands in [DOCUMENTATION.md](./DOCUMENTATION.md).
-
-Changelog of key features is available in [CHANGELOG.md](./CHANGELOG.md)
-
-## Development
-
-Development tools needed are:
-
-  * Latest Node 16 LTS release
-  * Docker
-  * Pre-commit
-
-### Pre-commit hooks
-
-Developers should use [pre-commit](https://pre-commit.com) on this repository to validate file formatting and commit
-messages on every commit. After pre-commit has been installed on your system, activate on the repository.
-
-```bash
-$ pre-commit install -t pre-commit -t commit-msg
-```
-
-### Get started
-
-After cloning the repository, install the dependencies.
-```bash
-$ npm install
+gcp-infra --help
 ```
 
-To run the CLI binary, link the project:
-```bash
-$ npm link
-```
-This will symlink the binary, making it possible to run like this:
-```bash
-$ gcp-infra
-```
+See detail documentation for all supported commands in [DOCUMENTATION.md](../../DOCUMENTATION.md).

package/bin/gcp-infra

@@ -1,4 +1,7 @@
 #!/usr/bin/env node
-const cli = require('../src/cli');
+const cli = require('../src/gcp-infra');
 
-cli(process.argv);
+cli.autoUpdate()
+  .then(() =>
+    cli.execute(process.argv),
+  );

package/package.json

@@ -1,23 +1,21 @@
 {
   "name": "@hiiretail/gcp-infra-cli",
-  "version": "0.104.0",
+  "version": "1.0.0",
   "description": "Infrastructure as code generator for GCP.",
-  "main": "src/cli.js",
+  "main": "src/gcp-infra.js",
   "bin": {
     "gcp-infra": "bin/gcp-infra",
     "@hiiretail/gcp-infra": "bin/gcp-infra"
   },
   "scripts": {
-    "lint:js": "eslint src generators __tests__",
-    "start": "node src/cli.js",
-    "test": "jest"
+    "start": "bin/gcp-infra"
   },
   "files": [
-    "generators",
-    "!generators/**/*.test.js",
+    "!src/*.test.js",
     "src"
   ],
   "repository": "github:extenda/gcp-infra-cli",
+  "readme": "https://github.com/extenda/gcp-infra-cli/tree/master/packages/gcp-infra#readme",
   "author": {
     "name": "Extenda Retail",
     "email": "info@extendaretail.com",
@@ -30,28 +28,7 @@
     "cli"
   ],
   "dependencies": {
-    "@google-cloud/storage": "^5.8.0",
-    "chalk": "^4.0.0",
-    "commander": "^5.1.0",
-    "enquirer-separator": "^0.1.0",
-    "glob": "^7.1.6",
-    "inquirer": "^7.3.3",
-    "js-yaml": "^4.1.0",
-    "npm-api": "^1.0.0",
-    "semver": "^7.3.2",
-    "sprintf-js": "^1.1.2",
-    "yeoman-environment": "^2.10.3",
-    "yeoman-generator": "^4.13.0"
-  },
-  "devDependencies": {
-    "eslint": "^6.8.0",
-    "eslint-config-airbnb-base": "^14.0.0",
-    "eslint-plugin-import": "^2.21.1",
-    "jest": "^27.5.1",
-    "jest-sonar": "^0.2.5",
-    "mock-fs": "^4.12.0",
-    "sprintf-js": "^1.1.2",
-    "yeoman-assert": "^3.1.1",
-    "yeoman-test": "^2.7.0"
+    "semver": "^7.7.2",
+    "tar": "^7.4.3"
   }
 }

package/src/gcp-infra.js

@@ -0,0 +1,87 @@
+const { existsSync, createWriteStream } = require('fs');
+const fs = require('fs/promises');
+const path = require('path');
+const { Readable } = require('stream');
+const { finished } = require('stream/promises');
+const tar = require('tar');
+const semver = require('semver');
+const home = require('os').homedir();
+const { version: cliVersion } = require('../package.json');
+
+const downloads = path.resolve(home, 'extenda', 'gcp-infra');
+const PROD_SOURCE = path.resolve(downloads, 'src');
+const DEV_SOURCE = path.resolve(__dirname, '..', '..', 'generators', 'src');
+
+const packageName = '@hiiretail/gcp-infra-generators';
+
+const downloadGenerators = async (version) => {
+  try {
+    await fs.rm(downloads, { recursive: true, force: true });
+    await fs.mkdir(downloads, { recursive: true });
+    const tarball = path.resolve(downloads, `generators-${version}.tgz`);
+    const registryName = packageName.replace('@hiiretail/', '');
+    const url = `https://registry.npmjs.org/${packageName}/-/${registryName}-${version}.tgz`;
+
+    const response = await fetch(url);
+    if (response.ok && response.body) {
+      await finished(
+        Readable.fromWeb(response.body).pipe(createWriteStream(tarball)),
+      );
+      await tar.x({ f: tarball, z: true, C: downloads, strip: 1 });
+      await fs.rm(tarball);
+      return `downloaded ${version}`;
+    }
+  } catch (err) {
+    console.error('Failed to download generators', err);
+  }
+  return 'download failed';
+};
+
+const getRegistryVersion = async () =>
+  fetch(`https://registry.npmjs.org/${packageName}`)
+    .then((response) => response.json())
+    .then((data) => {
+      const {
+        'dist-tags': { latest },
+      } = data;
+      return latest;
+    });
+
+const clientSource = () => [DEV_SOURCE, PROD_SOURCE].find(existsSync);
+
+const autoUpdate = async (source = clientSource()) => {
+  if (!source) {
+    console.log(`Installing ${packageName} from npmjs.org`);
+    return downloadGenerators(await getRegistryVersion());
+  } else if (source === PROD_SOURCE) {
+    const registryVersion = await getRegistryVersion();
+    const { version: installedVersion } = JSON.parse(
+      await fs.readFile(path.resolve(source, '..', 'package.json'), 'utf8'),
+    );
+    if (semver.gt(registryVersion, installedVersion)) {
+      console.log(
+        `Updating ${packageName}@${installedVersion} to ${registryVersion}`,
+      );
+      return downloadGenerators(registryVersion);
+    }
+  }
+  return 'no update';
+};
+
+const execute = async (argv, exitOverride) => {
+  const source = clientSource();
+  if (source) {
+    const modulePath = path.resolve(source, 'cli');
+    if (existsSync(`${modulePath}.js`)) {
+      await require(modulePath)(argv, exitOverride, cliVersion);
+    }
+  } else {
+    console.log('No generators found!');
+  }
+};
+
+module.exports = {
+  productionSource: () => PROD_SOURCE,
+  autoUpdate,
+  execute,
+};

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2020 Extenda Retail
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/generators/clan-resources/clan-project/generator.json

@@ -1,4 +0,0 @@
-{
-  "name": "Clan project with GKE Autopilot",
-  "description": "Create resources for clan project with GKE cluster"
-}

package/generators/clan-resources/clan-project/index.js

@@ -1,135 +0,0 @@
-const path = require('path');
-const fs = require('fs');
-const chalk = require('chalk');
-const BaseGenerator = require('../../../src/BaseGenerator');
-const { required } = require('../../../src/validators');
-
-module.exports = class extends BaseGenerator {
-  constructor(args, opts) {
-    super(args, opts);
-
-    // Output directory relative to the tf-infra-gcp directory.
-    this.destinationRoot('organization');
-  }
-
-  prompting() {
-    const prompts = [
-      {
-        type: 'list',
-        name: 'domain',
-        default: 'extendaretail-com',
-        message: 'Choose GCP Organization',
-        store: true,
-        choices: this.listSubDirectories(this.destinationPath()),
-        validate: required,
-      },
-      {
-        type: 'list',
-        name: 'department',
-        message: 'Choose department',
-        store: true,
-        choices: (answers) => this.listSubDirectories(
-          path.join(this.destinationPath(), answers.domain, 'departments'),
-        ),
-        validate: required,
-      },
-      {
-        type: 'list',
-        name: 'tribe',
-        message: 'Choose tribe',
-        store: true,
-        choices: (answers) => this.listSubDirectories(
-          path.join(this.destinationPath(), answers.domain, 'departments', answers.department),
-        ),
-        validate: required,
-      },
-      {
-        type: 'list',
-        name: 'clan',
-        message: 'Choose clan',
-        store: true,
-        choices: (answers) => this.listSubDirectories(
-          path.join(this.destinationPath(), answers.domain, 'departments', answers.department, answers.tribe, 'clans'),
-        ),
-        validate: required,
-      },
-      {
-        type: 'list',
-        name: 'migration',
-        message: 'Is the project in the process of migration to Autopilot?',
-        default: 'yes',
-        choices: ['yes', 'no'],
-      },
-    ];
-
-    return this.prompt(prompts).then((props) => {
-      this.answers = props;
-    });
-  }
-
-  writing() {
-    const {
-      domain,
-      department,
-      tribe,
-      clan,
-      migration,
-    } = this.answers;
-
-    ['prod', 'staging'].forEach((env) => {
-      this.copyDir(
-        'clan-resources',
-        path.join(domain, 'departments', department, tribe, 'clans', clan, env, 'resources'),
-        {
-          ...this.answers,
-          env,
-        },
-      );
-    });
-
-    if (migration === 'yes') {
-      ['prod', 'staging'].forEach((env) => {
-        const terragruntFilePath = path.join(domain, 'departments', department, tribe, 'clans', clan, env, 'terragrunt.hcl');
-
-        fs.readFile(terragruntFilePath, 'utf8', (err, data) => {
-          if (err) {
-            return;
-          }
-
-          const modifiedData = data.replace(
-            /source\s*=\s*"git::https:\/\/github\.com\/extenda\/tf-module-gcp-project\/\/\?ref=.+"/,
-            'source = "git::https://github.com/extenda/tf-module-gcp-project//?ref=migration"',
-          );
-
-          fs.writeFile(terragruntFilePath, modifiedData, 'utf8', (error) => {
-            if (error) {
-              console.error(`Error writing file ${terragruntFilePath}: ${error}`);
-            } else {
-              console.log(`Updated file ${terragruntFilePath}`);
-            }
-          });
-        });
-      });
-    }
-  }
-
-  end() {
-    const {
-      domain,
-      department,
-      tribe,
-      clan,
-    } = this.answers;
-    const clanDir = path.join(domain, 'departments', department, tribe, 'clans', clan);
-    this.log(`
-${chalk.green(`Your clan resources have now been created. To finalize your configuration, please continue
-with manual editing of the generated files.`)}
-${chalk.green('1.')} Review the configurations
-   \u2192 ${chalk.cyan(path.join(clanDir, 'prod', 'network', 'spec.hcl'))}
-   \u2192 ${chalk.cyan(path.join(clanDir, 'prod', 'gke', 'spec.hcl'))}
-   \u2192 ${chalk.cyan(path.join(clanDir, 'staging', 'network', 'spec.hcl'))}
-   \u2192 ${chalk.cyan(path.join(clanDir, 'staging', 'gke', 'spec.hcl'))}
-${chalk.green('2.')} Push this change in a feature branch and open a pull request.
-`);
-  }
-};

package/generators/clan-resources/clan-project/templates/clan-resources/cloud-nat/terragrunt.hcl

@@ -1,52 +0,0 @@
-# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
-# working directory, into a temporary folder, and execute your Terraform commands in that folder.
-terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-nat//?ref=v5.2.0"
-}
-
-# Include all settings from the root terragrunt.hcl file
-include {
-  path = find_in_parent_folders("terragrunt_root.hcl")
-}
-
-dependency "network" {
-  config_path = "../network"
-  mock_outputs = {
-    network_name = "network-dummy-name"
-    project_id   = "dummy-id"
-  }
-}
-
-dependency "router" {
-  config_path = "../cloud-router"
-  mock_outputs = {
-    router = {
-      name = "router-dummy-name"
-    }
-  }
-}
-
-dependency "static_ip" {
-  config_path = "../cloud-nat-static-ip"
-  mock_outputs = {
-    names = ["ip-name"]
-  }
-}
-
-locals {
-  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
-}
-
-# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
-inputs = merge(
-  {
-    name       = "${local.clan_name}-cloud-nat"
-    region     = "europe-west1"
-    project_id = dependency.network.outputs.project_id
-    network    = dependency.network.outputs.network_name
-    router     = dependency.router.outputs.router.name
-    nat_ips    = dependency.static_ip.outputs.names
-
-    enable_endpoint_independent_mapping = false
-  }
-)

package/generators/clan-resources/clan-project/templates/clan-resources/cloud-nat-static-ip/spec.hcl

@@ -1,7 +0,0 @@
-locals {
-  address_type = "EXTERNAL"
-  region       = "europe-west1"
-  names = [
-    "cloud-nat-static-ip",
-  ]
-}

package/generators/clan-resources/clan-project/templates/clan-resources/cloud-nat-static-ip/terragrunt.hcl

@@ -1,25 +0,0 @@
-terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-address//?ref=v4.0.0"
-}
-
-dependency "project" {
-  config_path = "../../"
-  mock_outputs = {
-    project_id = "project-dummy-id"
-  }
-}
-
-include {
-  path = find_in_parent_folders("terragrunt_root.hcl")
-}
-
-locals {
-  spec_vars = read_terragrunt_config("${get_terragrunt_dir()}/spec.hcl")
-}
-
-inputs = merge(
-  local.spec_vars.locals,
-  {
-    project_id = dependency.project.outputs.project_id
-  }
-)

package/generators/clan-resources/clan-project/templates/clan-resources/cloud-router/terragrunt.hcl

@@ -1,34 +0,0 @@
-terraform {
-  source = "git::https://github.com/terraform-google-modules/terraform-google-cloud-router//?ref=v6.0.2"
-}
-
-dependency "project" {
-  config_path = "../../"
-  mock_outputs = {
-    project_id = "project-dummy-id"
-  }
-}
-
-dependency "network" {
-  config_path = "../network"
-  mock_outputs = {
-    network_name = "network-dummy-name"
-  }
-}
-
-include {
-  path = find_in_parent_folders("terragrunt_root.hcl")
-}
-
-locals {
-  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
-}
-
-inputs = merge(
-  {
-    name    = "${local.clan_name}-router"
-    region  = "europe-west1"
-    project = dependency.project.outputs.project_id
-    network = dependency.network.outputs.network_name
-  }
-)

package/generators/clan-resources/clan-project/templates/clan-resources/gke/gke-resources/terragrunt.hcl

@@ -1,40 +0,0 @@
-terraform {
-  source = "git::https://github.com/extenda/tf-module-gcp-project//modules/gke-resources?ref=v2.0.0"
-}
-
-dependency "project" {
-  config_path = "../../../"
-  mock_outputs = {
-    project_id = "project-dummy-id"
-    ci_cd_service_account_email =  {
-      "ci-cd-pipeline" = "dummy"
-    }
-  }
-}
-
-dependency "gke" {
-  config_path = "../"
-  mock_outputs = {
-    endpoint       = "dummy-endpoint"
-    ca_certificate = base64encode("dummy-ca-certificate")
-   }
-}
-
-include {
-  path = find_in_parent_folders("terragrunt_root.hcl")
-}
-
-locals {
-  clan_name    = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
-  project_yaml = yamldecode(file(find_in_parent_folders("project.yaml")))
-}
-
-inputs = merge(
-  {
-    services           = local.project_yaml.services
-    project_id         = dependency.project.outputs.project_id
-    gke_host           = dependency.gke.outputs.endpoint
-    gke_ca_certificate = dependency.gke.outputs.ca_certificate
-    cicd_service       = dependency.project.outputs.ci_cd_service_account_email["ci-cd-pipeline"]
-  }
-)

package/generators/clan-resources/clan-project/templates/clan-resources/gke/spec.hcl

@@ -1,18 +0,0 @@
-locals {
-  regional                        = true
-  region                          = "europe-west1"
-  zones                           = ["europe-west1-d", "europe-west1-c", "europe-west1-b"]
-  ip_range_pods                   = "pods-cidr"
-  ip_range_services               = "services-cidr"
-  create_service_account          = true
-  grant_registry_access           = true
-  http_load_balancing             = true
-  add_cluster_firewall_rules      = false #needs to be changed to true after the first apply
-  dns_cache                       = true
-  enable_vertical_pod_autoscaling = true
-  enable_private_nodes            = true
-  master_global_access_enabled    = false
-  release_channel                 = "REGULAR"
-  enable_cost_allocation          = true
-  maintenance_start_time          = "01:00" #UTC
-}

package/generators/clan-resources/clan-project/templates/clan-resources/gke/terragrunt.hcl

@@ -1,52 +0,0 @@
-# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
-# working directory, into a temporary folder, and execute your Terraform commands in that folder.
-terraform {
-    # temp solution, change to the latest release once this is merged
-    source = "git::https://github.com/pziggo/terraform-google-kubernetes-engine//modules/beta-autopilot-private-cluster?ref=pziggo_autopilot_dns_config"
-}
-
-# Include all settings from the root terragrunt.hcl file
-include {
-  path = find_in_parent_folders("terragrunt_root.hcl")
-}
-
-dependency "project" {
-  config_path = "../../"
-  mock_outputs = {
-    project_id = "project-dummy-id"
-  }
-}
-
-dependency "network" {
-  config_path = "../network"
-  mock_outputs = {
-    network_name = "network-dummy-name"
-  }
-}
-
-locals {
-  spec_vars  = read_terragrunt_config("${get_terragrunt_dir()}/spec.hcl")
-  tribe_yaml = yamldecode(file(find_in_parent_folders("tribe.yaml")))
-  clan_name = basename(dirname(dirname(dirname(get_terragrunt_dir()))))
-  env_name = basename(dirname(dirname(get_terragrunt_dir())))
-}
-
-# These are the variables we have to pass in to use the module specified in the terragrunt configuration above
-inputs = merge(
-  local.spec_vars.locals,
-  {
-    name                         = "${local.clan_name}-cluster-${local.env_name}"
-    project_id                   = dependency.project.outputs.project_id
-    network                      = dependency.network.outputs.network_name
-    subnetwork                   = "k8s-subnet"
-    identity_namespace           = "${dependency.project.outputs.project_id}.svc.id.goog"
-    authenticator_security_group = "gke-security-groups@extendaretail.com"
-    registry_project_ids         = ["extenda"]
-    cluster_resource_labels      = {
-      cc = local.tribe_yaml.costcenter
-    }
-
-    // Binary Auth must be Enabled manually in this console - not supported by the module yet
-    enable_binary_authorization  = true
-  }
-)

package/generators/clan-resources/clan-project/templates/clan-resources/network/firewall-rules/config/main.tf

@@ -1,41 +0,0 @@
-resource "google_compute_firewall" "dataflow-workers" {
-  name    = "allow-dataflow-workers"
-  network = var.network
-  project = var.project
-
-  allow {
-    protocol = "tcp"
-    ports    = ["12345-12346"]
-  }
-
-  priority    = 0
-  direction   = "INGRESS"
-  source_tags = ["dataflow"]
-  target_tags = ["dataflow"]
-}
-
-resource "google_compute_firewall" "allow_negs" {
-  name    = "allow-negs"
-  network = var.network
-  project = var.project
-
-  allow {
-    protocol = "all"
-  }
-  source_ranges = ["35.191.0.0/16", "130.211.0.0/22"]
-  priority      = 1000
-  direction     = "INGRESS"
-}
-
-resource "google_compute_firewall" "allow_internal_lb" {
-  name    = "allow-internal-lb"
-  network = var.network
-  project = var.project
-
-  allow {
-    protocol = "all"
-  }
-  source_ranges = ["10.10.0.0/23"]
-  priority      = 1000
-  direction     = "INGRESS"
-}