polyn-cli 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f27c81897f9c1970d3b228466c1f0b5f34b5f6c5c1286064e06ff7106e72e7
-  data.tar.gz: f4aadba2632bfbc52aa066ac30f627de3a82af17651c1777e4cb4c07275ca967
+  metadata.gz: 4ae34f6e95adeb45fa4c421f9eb70960bf599e570a2934365efff5218948d954
+  data.tar.gz: f72f8e51082c3d68956bd28a78667d6e484e89c7e90aa87dc212ad6c1d7a59e5
 SHA512:
-  metadata.gz: 6dfd33b774a154d24d54ccc63843633343832ca536a0feb69f677b8373bccebd6d4fde76fffccea26e1387644667da904999b1f4058672a89dfd89fd9b44a7ee
-  data.tar.gz: 5744ef8f8f8db2e72a61b087f96c49db372217aee7964860485d5cb6aec65a4772ffc9a0f19e1e0f9fe2ae2eef0df9e63fa684b2408479e09939f1a2a53af2fe
+  metadata.gz: cf5e0ff4dae1a5769da3f041f32a579a159c7fe00281d8a8f563e0c0642d55a3632010f47d7351b6e72592cfd781e6eea1d88091a3a51dd12c8e9045809ca040
+  data.tar.gz: 9a91d45f22c7c2ec3d3dce6e32776607d9da6d17d6c93a811c5219e4ae11cf0b47aa60bd0615fb705d857fe1fcd60df3e20243a92c4cfeb98e8170dfa892c0d6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyn-cli (0.1.1)
+    polyn-cli (0.1.2)
       dotenv (~> 2.7.6)
       json_schemer (~> 0.2)
       nats-pure (~> 2.0.0)

data/README.md

@@ -33,6 +33,7 @@ Run `polyn up` to update your NATS server with the latest configuration in your
 ## Environment Variables
 
 * `NATS_SERVERS` - locations of your servers (defaults to localhost)
+* `NATS_CREDENTIALS` - path to nats credentials file
 * `POLYN_ENV` - type of environment (defaults to "development")
 
 ## Development

data/lib/polyn/cli/configuration.rb

@@ -5,11 +5,12 @@ module Polyn
     ##
     # Configuration data for Polyn::Cli
     class Configuration
-      attr_reader :polyn_env, :nats_servers
+      attr_reader :polyn_env, :nats_servers, :nats_credentials
 
       def initialize
-        @polyn_env    = ENV["POLYN_ENV"] || "development"
-        @nats_servers = ENV["NATS_SERVERS"] || "localhost:4222"
+        @polyn_env        = ENV["POLYN_ENV"] || "development"
+        @nats_servers     = ENV["NATS_SERVERS"] || "localhost:4222"
+        @nats_credentials = ENV["NATS_CREDENTIALS"] || ""
       end
     end
   end

data/lib/polyn/cli/schema_generator.rb

@@ -14,21 +14,32 @@ module Polyn
 
       source_root File.join(File.expand_path(__dir__), "../templates")
 
+      def type
+        @type ||= event_type.split("/").last
+      end
+
+      def subdir
+        @subdir ||= begin
+          split = event_type.split("/") - [type]
+          split.join("/")
+        end
+      end
+
       def check_name
-        Polyn::Cli::Naming.validate_event_type!(event_type)
+        Polyn::Cli::Naming.validate_event_type!(type)
       end
 
       def file_name
-        @file_name ||= "#{event_type}.json"
+        @file_name ||= File.join(subdir, "#{type}.json")
       end
 
       def schema_id
-        Polyn::Cli::Naming.dot_to_colon(event_type)
+        Polyn::Cli::Naming.dot_to_colon(type)
       end
 
       def create
-        say "Creating new schema for #{event_type}"
-        template "generators/schema.json", File.join(options.dir, "events/#{event_type}.json")
+        say "Creating new schema for #{file_name}"
+        template "generators/schema.json", File.join(options.dir, "events/#{file_name}")
       end
     end
   end

data/lib/polyn/cli/schema_loader.rb

@@ -41,7 +41,10 @@ module Polyn
       attr_reader :thor, :events, :client, :bucket, :cloud_event_schema, :events_dir
 
       def read_events
-        Dir.glob(File.join(events_dir, "*.json")).each do |event_file|
+        event_files = Dir.glob(File.join(events_dir, "/**/*.json"))
+        validate_unique_event_types!(event_files)
+
+        event_files.each do |event_file|
           thor.say "Loading 'event #{event_file}'"
           data_schema = JSON.parse(File.read(event_file))
           event_type  = File.basename(event_file, ".json")
@@ -53,6 +56,30 @@ module Polyn
         end
       end
 
+      def validate_unique_event_types!(event_files)
+        duplicates = find_duplicates(event_files)
+        unless duplicates.empty?
+          messages = duplicates.reduce([]) do |memo, (event_type, files)|
+            memo << [event_type, *files].join("\n")
+          end
+          message  = [
+            "There can only be one of each event type. The following events were duplicated:",
+            *messages,
+          ].join("\n")
+          raise Polyn::Cli::ValidationError, message
+        end
+      end
+
+      def find_duplicates(event_files)
+        event_types = event_files.group_by do |event_file|
+          File.basename(event_file, ".json")
+        end
+        event_types.each_with_object({}) do |(event_type, files), hash|
+          hash[event_type] = files if files.length > 1
+          hash
+        end
+      end
+
       def validate_schema!(event_type, schema)
         JSONSchemer.schema(schema)
       rescue StandardError => e

data/lib/polyn/cli/version.rb

@@ -2,6 +2,6 @@
 
 module Polyn
   class Cli
-    VERSION = "0.1.1"
+    VERSION = "0.1.2"
   end
 end

data/lib/polyn/cli.rb

@@ -54,8 +54,11 @@ module Polyn
         directory "tf", File.join(options.dir, "tf")
         directory "events", File.join(options.dir, "events")
         template "docker-compose.yml", File.join(options.dir, "docker-compose.yml")
-        template "gitignore", File.join(options.dir, ".gitignore")
+        template "Dockerfile", File.join(options.dir, "Dockerfile")
+        template ".dockerignore", File.join(options.dir, ".dockerignore")
+        template ".gitignore", File.join(options.dir, ".gitignore")
         template "README.md", File.join(options.dir, "README.md")
+        template "Gemfile", File.join(options.dir, "Gemfile")
         run tf_init
         say "Initializing git"
         inside options.dir do
@@ -67,22 +70,44 @@ module Polyn
       method_option :dir, default: Dir.getwd
       desc "tf_init", "Initializes Terraform for configuration"
       def tf_init
+        terraform_root = File.join(options.dir, "tf")
         say "Initializing Terraform"
-        inside File.join(options.dir, "tf") do
-          run "terraform init"
+        inside terraform_root do
+          # In a development environment we want developers to work with their own local
+          # .tfstate rather than one configured in a remote `backend` intended for
+          # production use.
+          # https://www.terraform.io/language/settings/backends/configuration
+          #
+          # Terraform assumes only one backend will be configured and there's no path
+          # to switch between local and remote. There's also no way to dynamically load
+          # modules. https://github.com/hashicorp/terraform/issues/1439
+          # Instead we'll copy a backend config to the terraform root if we're in a production
+          # environment
+          if polyn_env == "production"
+            add_remote_backend(terraform_root) { run "terraform init" }
+          else
+            run "terraform init"
+          end
         end
       end
 
       desc "up", "updates the JetStream streams and consumers, as well the Polyn event registry"
       def up
-        if polyn_env == "development"
+        terraform_root = File.join(Dir.getwd, "tf")
+        # We only want to run nats in the docker container if
+        # the developer isn't already running nats themselves locally
+        if polyn_env == "development" && !nats_running?
           say "Starting NATS"
           run "docker compose up --detach"
         end
 
         say "Updating JetStream configuration"
         inside "tf" do
-          run tf_apply
+          if polyn_env == "production"
+            add_remote_backend(terraform_root) { run tf_apply }
+          else
+            run tf_apply
+          end
         end
 
         say "Updating Polyn event registry"
@@ -99,14 +124,35 @@ module Polyn
         Polyn::Cli.configuration.nats_servers
       end
 
+      def nats_credentials
+        Polyn::Cli.configuration.nats_credentials
+      end
+
       def tf_apply
         if polyn_env == "development"
-          %(terraform apply -var "jetstream_servers=#{nats_servers}" -auto-approve)
-        else
           %(terraform apply -var "jetstream_servers=#{nats_servers}")
+        else
+          "terraform apply -auto-approve -input=false "\
+            "-var \"jetstream_servers=#{nats_servers}\" "\
+            "-var \"nats_credentials=#{nats_credentials}\" " \
+            "-var \"polyn_env=production\""
         end
       end
 
+      def nats_running?
+        # Uses lsof command to look up a process id. Will return `true` if it finds one
+        system("lsof -i TCP:4222 -t")
+      end
+
+      def add_remote_backend(tf_root)
+        copy_file File.join(tf_root, "remote_state_config/backend.tf"), "backend.tf"
+        yield
+      # We always want to remove the backend.tf file even if there's an error
+      # this way you don't get into a weird state when testing locally
+      ensure
+        remove_file File.join(tf_root, "backend.tf")
+      end
+
       register(Polyn::Cli::SchemaGenerator, "gen:schema", "gen:schema EVENT_TYPE",
         "Generates a new JSON Schema file for an event")
       register(Polyn::Cli::StreamGenerator, "gen:stream", "gen:stream NAME",

data/lib/polyn/templates/.dockerignore

@@ -0,0 +1,6 @@
+.git
+tf/.terraform
+tf/*.tfstate
+tf/*.backup
+# We don't ignore the terraform lockfile because it ensures that
+# the provider dependency versions are consistent

data/lib/polyn/templates/.gitignore

@@ -0,0 +1,5 @@
+tf/.terraform/**
+*.tfstate
+*.backup
+# We don't ignore the terraform lockfile because it ensures that
+# the provider dependency versions are consistent

data/lib/polyn/templates/Dockerfile

@@ -0,0 +1,20 @@
+FROM ruby:3.0.4-alpine3.15 as base
+RUN apk add terraform
+ADD Gemfile* ./
+RUN gem install bundler
+RUN bundle install
+
+FROM base as app
+WORKDIR /events
+ADD events ./events
+ADD tf ./tf
+
+FROM app as dev
+ENV POLYN_ENV='development'
+RUN bundle exec polyn tf_init
+CMD bundle exec polyn up
+
+FROM app as prod
+ENV POLYN_ENV='production'
+RUN bundle exec polyn tf_init
+CMD bundle exec polyn up

data/lib/polyn/templates/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "polyn-cli", "~> <%= Polyn::Cli::VERSION %>"

data/lib/polyn/templates/README.md

@@ -9,10 +9,19 @@ environment.
 2. Install Terraform. For M1 Macs, [download the AMD64 version](https://www.terraform.io/downloads)
    rather than using Homebrew to install Terraform.
 3. Ensure Docker & Docker Compose is installed
-4. [Install the Polyn CLI]()
-5. Call `polyn up`. By default this will run in `development` mode, which will start the NATS
+4. [Install the Polyn CLI](https://github.com/SpiffInc/polyn-cli)
+5. Call `polyn tf_init` if this is the first time using terraform in the codebase.
+6. Call `polyn up`. By default this will run in `development` mode, which will start the NATS
    server, configure it via Terraform, and update the Polyn Event Registry.
 
+### Running NATS locally
+
+`polyn up` will use run a Docker container for you if one is not already running. Alternatively, you can run `nats-server` yourself locally if you prefer.
+
+## Naming Conventions
+
+See the Protocol Documentation for [Naming Conventions](https://github.com/SpiffInc/polyn-protocol/blob/main/NAMING_CONVENTIONS.md)
+
 ## Streams
 
 Each stream should have its own configuration file under `./tf` . Run `polyn gen:stream <stream_name>` to generate a new configuration file for a stream
@@ -32,6 +41,12 @@ Every schema should be a valid [JSON Schema](https://json-schema.org/) document.
 The Polyn CLI tool will combine your event schema with the [Cloud Events Schema](https://cloudevents.io/) when it adds it to the Polyn Event Registry.
 This means you only need to include the JSON Schema for the `data` portion of the Cloud Event and not the entire Cloud Event schema.
 
+### Subdirectories
+
+If you'd like to organize your events by team ownership or some other convention, you can use subdirectories to do so. The full event type should still be part of the file name. You should also ensure there are not duplicate event types in different directories as only one schema can be defined per event type.
+
+You can generate a schema in a subdirectory like this: `polyn gen:schema some/nested/dir/widgets.created.v1`
+
 ## Schema Versioning
 
 ### New Event
@@ -49,3 +64,21 @@ Making a change to an event schema that is not backwards-compatible will require
 json file. The new file should have the same name as your old file, but with the version number increased. Your
 Producers will need to continue producing both events until you are sure there are no more consumers using the
 old event.
+
+## Terraform State
+
+Terraform generates and maintains a [`terraform.tfstate`](https://www.terraform.io/language/state) file that is used to map terraform configuration to real production server instances. Polyn needs to interact with this file differently based on whether we are developing locally or in a production environment.
+
+### Local Development
+
+For local development Polyn expects the `terraform.tfstate` file to exist in the local file system. However, it should not be checked in to version control. We don't want experiments and updates made on a local developer machines to end up as the "source of truth" for our production infrastucture.
+
+### Production
+
+In production Terraform recommends keeping `terraform.tfstate` in a [remote storage location](https://www.terraform.io/language/state). The remote state file should be the "source of truth" for your infrastucture and shouldn't be getting accessed during development. Depending on the size of your organization and security policies, not all developers will have access to the remote storage source and you don't want that to prohibit them from adding events, streams, or consumers.
+
+Polyn expects you to keep a `./remote_state_config/backend.tf` file that configures a Terraform [backend](https://www.terraform.io/language/settings/backends/configuration). This will only be used when `POLYN_ENV=production`.
+
+## Deployment
+
+The default `Dockerfile` generated by [Install the Polyn CLI](https://github.com/SpiffInc/polyn-cli) can help you create an image that will configure terraform and your infrastucture with the latest changes when you execute `docker run`. You'll need to pass in `env` variables for `NATS_SERVERS` and `NATS_CREDENTIALS`. Also any `env` variables needed to connect to your remote state storage.

data/lib/polyn/templates/tf/provider.tf

@@ -0,0 +1,4 @@
+provider "jetstream" {
+  servers = var.jetstream_servers
+  credentials = var.nats_credentials
+}

data/lib/polyn/templates/tf/remote_state_config/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  // Configure a [backend](https://www.terraform.io/language/settings/backends/configuration)
+  // to store your `terraform.tfstate` file in for production use
+  backend "remote" {
+  }
+}

data/lib/polyn/templates/tf/variables.tf

@@ -0,0 +1,15 @@
+variable "jetstream_servers" {
+  type = string
+  description = "The JetStream servers to connect to"
+}
+
+variable "nats_credentials" {
+  type = string
+  description = "Path to file with NATS credentials"
+}
+
+variable "polyn_env" {
+  type = string
+  description = "The environment terraform is running in"
+  default = "development"
+}

data/lib/polyn/templates/tf/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    jetstream = {
+      source = "nats-io/jetstream"
+      version = "~> 0.0.31"
+    }
+  }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyn-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Jarod
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-10 00:00:00.000000000 Z
+date: 2022-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -102,15 +102,21 @@ files:
 - lib/polyn/cli/stream_generator.rb
 - lib/polyn/cli/version.rb
 - lib/polyn/cloud-event-schema.json
+- lib/polyn/templates/.dockerignore
+- lib/polyn/templates/.gitignore
+- lib/polyn/templates/Dockerfile
+- lib/polyn/templates/Gemfile
 - lib/polyn/templates/README.md
 - lib/polyn/templates/docker-compose.yml
 - lib/polyn/templates/events/.gitkeep
 - lib/polyn/templates/events/widgets.created.v1.json
 - lib/polyn/templates/generators/schema.json
 - lib/polyn/templates/generators/stream.tf
-- lib/polyn/templates/gitignore
-- lib/polyn/templates/tf/event_registry.tf
-- lib/polyn/templates/tf/main.tf
+- lib/polyn/templates/tf/kv_buckets.tf
+- lib/polyn/templates/tf/provider.tf
+- lib/polyn/templates/tf/remote_state_config/backend.tf
+- lib/polyn/templates/tf/variables.tf
+- lib/polyn/templates/tf/versions.tf
 - lib/polyn/templates/tf/widgets.tf
 - polyn-cli.gemspec
 homepage: https://github.com/Spiffinc/polyn-cli

data/lib/polyn/templates/gitignore

@@ -1,3 +0,0 @@
-tf/.terraform/**
-*.tfstate
-.terraform.lock.hcl

data/lib/polyn/templates/tf/main.tf

@@ -1,17 +0,0 @@
-terraform {
-  required_providers {
-    jetstream = {
-      source = "nats-io/jetstream"
-    }
-  }
-
-}
-
-variable "jetstream_servers" {
-  type = string
-  description = "The JetStream servers to connect to"
-}
-
-provider "jetstream" {
-  servers = var.jetstream_servers
-}