polyn-cli 0.1.1 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6f27c81897f9c1970d3b228466c1f0b5f34b5f6c5c1286064e06ff7106e72e7
-  data.tar.gz: f4aadba2632bfbc52aa066ac30f627de3a82af17651c1777e4cb4c07275ca967
+  metadata.gz: a516676ead604b6dcfc242c75addef95ad9512f4c7a9b009369cb46d581f946e
+  data.tar.gz: 9ebdfaec0ba3df014121d75d9903ee234dad27ba4444d45178cdf66fd132336d
 SHA512:
-  metadata.gz: 6dfd33b774a154d24d54ccc63843633343832ca536a0feb69f677b8373bccebd6d4fde76fffccea26e1387644667da904999b1f4058672a89dfd89fd9b44a7ee
-  data.tar.gz: 5744ef8f8f8db2e72a61b087f96c49db372217aee7964860485d5cb6aec65a4772ffc9a0f19e1e0f9fe2ae2eef0df9e63fa684b2408479e09939f1a2a53af2fe
+  metadata.gz: 545df32cdd3d928022e23798cbb08ee88521fc54248a1fd1911e80b88ff71d97987f092b2d73429942ec2fa66e818a8c0d88d11de30d7f5f77bd73ff3078e496
+  data.tar.gz: 58f0c9aaa9348efdf8c4310c1c1e7376cfb57d43e2f662015cbc224d14df0306ffc6da439f0618a079d08f57aeb3b08391b61cfa622ea5d1f8843fbd07989623

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    polyn-cli (0.1.1)
+    polyn-cli (0.1.4)
       dotenv (~> 2.7.6)
       json_schemer (~> 0.2)
       nats-pure (~> 2.0.0)

data/README.md

@@ -33,6 +33,7 @@ Run `polyn up` to update your NATS server with the latest configuration in your
 ## Environment Variables
 
 * `NATS_SERVERS` - locations of your servers (defaults to localhost)
+* `NATS_CREDENTIALS` - path to nats credentials file
 * `POLYN_ENV` - type of environment (defaults to "development")
 
 ## Development

data/bin/apply_changes.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# We want to keep these commands out of the Docker image
+# build and instead defer them till the container is run.
+# terraform init may require credentials for a remote backend
+# and we don't want that in the image for security reasons.
+bundle exec polyn tf_init
+bundle exec polyn up

data/lib/polyn/cli/configuration.rb

@@ -5,11 +5,12 @@ module Polyn
     ##
     # Configuration data for Polyn::Cli
     class Configuration
-      attr_reader :polyn_env, :nats_servers
+      attr_reader :polyn_env, :nats_servers, :nats_credentials
 
       def initialize
-        @polyn_env    = ENV["POLYN_ENV"] || "development"
-        @nats_servers = ENV["NATS_SERVERS"] || "localhost:4222"
+        @polyn_env        = ENV["POLYN_ENV"] || "development"
+        @nats_servers     = ENV["NATS_SERVERS"] || "localhost:4222"
+        @nats_credentials = ENV["NATS_CREDENTIALS"] || ""
       end
     end
   end

data/lib/polyn/cli/schema_generator.rb

@@ -14,21 +14,32 @@ module Polyn
 
       source_root File.join(File.expand_path(__dir__), "../templates")
 
+      def type
+        @type ||= event_type.split("/").last
+      end
+
+      def subdir
+        @subdir ||= begin
+          split = event_type.split("/") - [type]
+          split.join("/")
+        end
+      end
+
       def check_name
-        Polyn::Cli::Naming.validate_event_type!(event_type)
+        Polyn::Cli::Naming.validate_event_type!(type)
       end
 
       def file_name
-        @file_name ||= "#{event_type}.json"
+        @file_name ||= File.join(subdir, "#{type}.json")
       end
 
       def schema_id
-        Polyn::Cli::Naming.dot_to_colon(event_type)
+        Polyn::Cli::Naming.dot_to_colon(type)
       end
 
       def create
-        say "Creating new schema for #{event_type}"
-        template "generators/schema.json", File.join(options.dir, "events/#{event_type}.json")
+        say "Creating new schema for #{file_name}"
+        template "generators/schema.json", File.join(options.dir, "events/#{file_name}")
       end
     end
   end

data/lib/polyn/cli/schema_loader.rb

@@ -19,29 +19,44 @@ module Polyn
       def initialize(thor, **opts)
         @thor               = thor
         @client             = NATS.connect(Polyn::Cli.configuration.nats_servers).jetstream
-        @bucket             = client.key_value(opts.fetch(:store_name, STORE_NAME))
+        @store_name         = opts.fetch(:store_name, STORE_NAME)
+        @bucket             = client.key_value(@store_name)
         @cloud_event_schema = Polyn::Cli::CloudEvent.to_h.freeze
         @events_dir         = opts.fetch(:events_dir, File.join(Dir.pwd, "events"))
         @events             = {}
+        @existing_events    = {}
       end
 
       def load_events
         thor.say "Loading events into the Polyn event registry from '#{events_dir}'"
         read_events
+        load_existing_events
 
         events.each do |name, event|
           bucket.put(name, JSON.generate(event))
         end
 
+        delete_missing_events
+
         true
       end
 
       private
 
-      attr_reader :thor, :events, :client, :bucket, :cloud_event_schema, :events_dir
+      attr_reader :thor,
+        :events,
+        :client,
+        :bucket,
+        :cloud_event_schema,
+        :events_dir,
+        :store_name,
+        :existing_events
 
       def read_events
-        Dir.glob(File.join(events_dir, "*.json")).each do |event_file|
+        event_files = Dir.glob(File.join(events_dir, "/**/*.json"))
+        validate_unique_event_types!(event_files)
+
+        event_files.each do |event_file|
           thor.say "Loading 'event #{event_file}'"
           data_schema = JSON.parse(File.read(event_file))
           event_type  = File.basename(event_file, ".json")
@@ -53,6 +68,30 @@ module Polyn
         end
       end
 
+      def validate_unique_event_types!(event_files)
+        duplicates = find_duplicates(event_files)
+        unless duplicates.empty?
+          messages = duplicates.reduce([]) do |memo, (event_type, files)|
+            memo << [event_type, *files].join("\n")
+          end
+          message  = [
+            "There can only be one of each event type. The following events were duplicated:",
+            *messages,
+          ].join("\n")
+          raise Polyn::Cli::ValidationError, message
+        end
+      end
+
+      def find_duplicates(event_files)
+        event_types = event_files.group_by do |event_file|
+          File.basename(event_file, ".json")
+        end
+        event_types.each_with_object({}) do |(event_type, files), hash|
+          hash[event_type] = files if files.length > 1
+          hash
+        end
+      end
+
       def validate_schema!(event_type, schema)
         JSONSchemer.schema(schema)
       rescue StandardError => e
@@ -68,6 +107,32 @@ module Polyn
           }),
         })
       end
+
+      def load_existing_events
+        sub = client.subscribe("#{key_prefix}.>")
+
+        loop do
+          msg                                                     = sub.next_msg
+          existing_events[msg.subject.gsub("#{key_prefix}.", "")] = msg.data
+        # A timeout is the only mechanism given to indicate there are no
+        # more messages
+        rescue NATS::IO::Timeout
+          break
+        end
+        sub.unsubscribe
+      end
+
+      def key_prefix
+        "$KV.#{store_name}"
+      end
+
+      def delete_missing_events
+        missing_events = existing_events.keys - events.keys
+        missing_events.each do |event|
+          thor.say "Deleting event #{event}"
+          bucket.delete(event)
+        end
+      end
     end
   end
 end

data/lib/polyn/cli/version.rb

@@ -2,6 +2,6 @@
 
 module Polyn
   class Cli
-    VERSION = "0.1.1"
+    VERSION = "0.1.4"
   end
 end

data/lib/polyn/cli.rb

@@ -54,8 +54,11 @@ module Polyn
         directory "tf", File.join(options.dir, "tf")
         directory "events", File.join(options.dir, "events")
         template "docker-compose.yml", File.join(options.dir, "docker-compose.yml")
-        template "gitignore", File.join(options.dir, ".gitignore")
+        template "Dockerfile", File.join(options.dir, "Dockerfile")
+        template ".dockerignore", File.join(options.dir, ".dockerignore")
+        template ".gitignore", File.join(options.dir, ".gitignore")
         template "README.md", File.join(options.dir, "README.md")
+        template "Gemfile", File.join(options.dir, "Gemfile")
         run tf_init
         say "Initializing git"
         inside options.dir do
@@ -67,22 +70,44 @@ module Polyn
       method_option :dir, default: Dir.getwd
       desc "tf_init", "Initializes Terraform for configuration"
       def tf_init
+        terraform_root = File.join(options.dir, "tf")
         say "Initializing Terraform"
-        inside File.join(options.dir, "tf") do
-          run "terraform init"
+        inside terraform_root do
+          # In a development environment we want developers to work with their own local
+          # .tfstate rather than one configured in a remote `backend` intended for
+          # production use.
+          # https://www.terraform.io/language/settings/backends/configuration
+          #
+          # Terraform assumes only one backend will be configured and there's no path
+          # to switch between local and remote. There's also no way to dynamically load
+          # modules. https://github.com/hashicorp/terraform/issues/1439
+          # Instead we'll copy a backend config to the terraform root if we're in a production
+          # environment
+          if polyn_env == "production"
+            add_remote_backend(terraform_root) { run "terraform init" }
+          else
+            run "terraform init"
+          end
         end
       end
 
       desc "up", "updates the JetStream streams and consumers, as well the Polyn event registry"
       def up
-        if polyn_env == "development"
+        terraform_root = File.join(Dir.getwd, "tf")
+        # We only want to run nats in the docker container if
+        # the developer isn't already running nats themselves locally
+        if polyn_env == "development" && !nats_running?
           say "Starting NATS"
           run "docker compose up --detach"
         end
 
         say "Updating JetStream configuration"
         inside "tf" do
-          run tf_apply
+          if polyn_env == "production"
+            add_remote_backend(terraform_root) { run tf_apply }
+          else
+            run tf_apply
+          end
         end
 
         say "Updating Polyn event registry"
@@ -99,14 +124,35 @@ module Polyn
         Polyn::Cli.configuration.nats_servers
       end
 
+      def nats_credentials
+        Polyn::Cli.configuration.nats_credentials
+      end
+
       def tf_apply
         if polyn_env == "development"
-          %(terraform apply -var "jetstream_servers=#{nats_servers}" -auto-approve)
+          %(terraform apply -auto-approve -input=false -var "jetstream_servers=#{nats_servers}")
         else
-          %(terraform apply -var "jetstream_servers=#{nats_servers}")
+          "terraform apply -auto-approve -input=false "\
+            "-var \"jetstream_servers=#{nats_servers}\" "\
+            "-var \"nats_credentials=#{nats_credentials}\" " \
+            "-var \"polyn_env=production\""
         end
       end
 
+      def nats_running?
+        # Uses lsof command to look up a process id. Will return `true` if it finds one
+        system("lsof -i TCP:4222 -t")
+      end
+
+      def add_remote_backend(tf_root)
+        copy_file File.join(tf_root, "remote_state_config/backend.tf"), "backend.tf"
+        yield
+      # We always want to remove the backend.tf file even if there's an error
+      # this way you don't get into a weird state when testing locally
+      ensure
+        remove_file File.join(tf_root, "backend.tf")
+      end
+
       register(Polyn::Cli::SchemaGenerator, "gen:schema", "gen:schema EVENT_TYPE",
         "Generates a new JSON Schema file for an event")
       register(Polyn::Cli::StreamGenerator, "gen:stream", "gen:stream NAME",

data/lib/polyn/templates/.dockerignore

@@ -0,0 +1,6 @@
+.git
+tf/.terraform
+tf/*.tfstate
+tf/*.backup
+# We don't ignore the terraform lockfile because it ensures that
+# the provider dependency versions are consistent

data/lib/polyn/templates/.gitignore

@@ -0,0 +1,5 @@
+tf/.terraform/**
+*.tfstate
+*.backup
+# We don't ignore the terraform lockfile because it ensures that
+# the provider dependency versions are consistent

data/lib/polyn/templates/Dockerfile

@@ -0,0 +1,16 @@
+FROM ruby:3.0.4-alpine3.15 as base
+RUN apk add terraform
+ADD Gemfile* ./
+RUN gem install bundler
+RUN bundle install
+
+FROM base as app
+WORKDIR /events
+ADD events ./events
+ADD tf ./tf
+
+FROM app as dev
+ENV POLYN_ENV='development'
+
+FROM app as prod
+ENV POLYN_ENV='production'

data/lib/polyn/templates/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gem "polyn-cli", "~> <%= Polyn::Cli::VERSION %>"

data/lib/polyn/templates/README.md

@@ -3,27 +3,35 @@
 This repository contains all of the events and terraform resources for the Polyn services
 environment.
 
-## Development Setup
-
 1. Install [Ruby](https://github.com/asdf-vm/asdf-ruby)
-2. Install Terraform. For M1 Macs, [download the AMD64 version](https://www.terraform.io/downloads)
+2. Install bundler `gem install bundler`
+3. Install dependencies `bundle install`
+4. Install Terraform. For M1 Macs, [download the AMD64 version](https://www.terraform.io/downloads)
    rather than using Homebrew to install Terraform.
-3. Ensure Docker & Docker Compose is installed
-4. [Install the Polyn CLI]()
-5. Call `polyn up`. By default this will run in `development` mode, which will start the NATS
+5. Ensure Docker & Docker Compose is installed
+6. Call `bundle exec polyn tf_init` if this is the first time using terraform in the codebase.
+7. Call `bundle exec polyn up`. By default this will run in `development` mode, which will start the NATS
    server, configure it via Terraform, and update the Polyn Event Registry.
 
+### Running NATS locally
+
+`bundle exec polyn up` will use run a Docker container for you if one is not already running. Alternatively, you can run `nats-server` yourself locally if you prefer.
+
+## Naming Conventions
+
+See the Protocol Documentation for [Naming Conventions](https://github.com/SpiffInc/polyn-protocol/blob/main/NAMING_CONVENTIONS.md)
+
 ## Streams
 
-Each stream should have its own configuration file under `./tf` . Run `polyn gen:stream <stream_name>` to generate a new configuration file for a stream
+Each stream should have its own configuration file under `./tf`. Run `bundle exec polyn gen:stream <stream_name>` to generate a new configuration file for a stream
 
 ## Consumers
 
-Run `polyn gen:consumer <stream_name> <destination_name> <event_type>` to generate new configuration for a consumer of a stream. It will be included in the same file as the stream configuration.
+Run `bundle exec polyn gen:consumer <stream_name> <destination_name> <event_type>` to generate new configuration for a consumer of a stream. It will be included in the same file as the stream configuration.
 
 ## Event Schemas
 
-Run `polyn gen:schema <event_type>` to generate a new JSON Schema for an event
+Run `bundle exec polyn gen:schema <event_type>` to generate a new JSON Schema for an event
 
 All the schemas for your events should live in the `./events` directory.
 The name of your schema file should be the same as your event, but with `.json` at the end.
@@ -32,6 +40,12 @@ Every schema should be a valid [JSON Schema](https://json-schema.org/) document.
 The Polyn CLI tool will combine your event schema with the [Cloud Events Schema](https://cloudevents.io/) when it adds it to the Polyn Event Registry.
 This means you only need to include the JSON Schema for the `data` portion of the Cloud Event and not the entire Cloud Event schema.
 
+### Subdirectories
+
+If you'd like to organize your events by team ownership or some other convention, you can use subdirectories to do so. The full event type should still be part of the file name. You should also ensure there are not duplicate event types in different directories as only one schema can be defined per event type.
+
+You can generate a schema in a subdirectory like this: `bundle exec polyn gen:schema some/nested/dir/widgets.created.v1`
+
 ## Schema Versioning
 
 ### New Event
@@ -49,3 +63,23 @@ Making a change to an event schema that is not backwards-compatible will require
 json file. The new file should have the same name as your old file, but with the version number increased. Your
 Producers will need to continue producing both events until you are sure there are no more consumers using the
 old event.
+
+## Terraform State
+
+Terraform generates and maintains a [`terraform.tfstate`](https://www.terraform.io/language/state) file that is used to map terraform configuration to real production server instances. Polyn needs to interact with this file differently based on whether we are developing locally or in a production environment.
+
+### Local Development
+
+For local development Polyn expects the `terraform.tfstate` file to exist in the local file system. However, it should not be checked in to version control. We don't want experiments and updates made on a local developer machines to end up as the "source of truth" for our production infrastucture.
+
+### Production
+
+In production Terraform recommends keeping `terraform.tfstate` in a [remote storage location](https://www.terraform.io/language/state). The remote state file should be the "source of truth" for your infrastucture and shouldn't be getting accessed during development. Depending on the size of your organization and security policies, not all developers will have access to the remote storage source and you don't want that to prohibit them from adding events, streams, or consumers.
+
+Polyn expects you to keep a `./remote_state_config/backend.tf` file that configures a Terraform [backend](https://www.terraform.io/language/settings/backends/configuration). This will only be used when `POLYN_ENV=production`.
+
+## Deployment
+
+The default `Dockerfile` generated by [Install the Polyn CLI](https://github.com/SpiffInc/polyn-cli) can help you create an image with the latest changes and nessary environment to run polyn commands.
+
+The `bin/apply_changes.sh` script can be used to execute the polyn commands you need to update your production NATS server. You'll need to pass in `env` variables for `NATS_SERVERS` and `NATS_CREDENTIALS`. Also any `env` variables needed to connect to your remote state storage.

data/lib/polyn/templates/tf/provider.tf

@@ -0,0 +1,4 @@
+provider "jetstream" {
+  servers = var.jetstream_servers
+  credentials = var.nats_credentials
+}

data/lib/polyn/templates/tf/remote_state_config/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  // Configure a [backend](https://www.terraform.io/language/settings/backends/configuration)
+  // to store your `terraform.tfstate` file in for production use
+  backend "remote" {
+  }
+}

data/lib/polyn/templates/tf/variables.tf

@@ -0,0 +1,15 @@
+variable "jetstream_servers" {
+  type = string
+  description = "The JetStream servers to connect to"
+}
+
+variable "nats_credentials" {
+  type = string
+  description = "Path to file with NATS credentials"
+}
+
+variable "polyn_env" {
+  type = string
+  description = "The environment terraform is running in"
+  default = "development"
+}

data/lib/polyn/templates/tf/versions.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    jetstream = {
+      source = "nats-io/jetstream"
+      version = "~> 0.0.31"
+    }
+  }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polyn-cli
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.4
 platform: ruby
 authors:
 - Jarod
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-08-10 00:00:00.000000000 Z
+date: 2022-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -89,6 +89,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/apply_changes.sh
 - bin/console
 - bin/setup
 - exe/polyn
@@ -102,15 +103,21 @@ files:
 - lib/polyn/cli/stream_generator.rb
 - lib/polyn/cli/version.rb
 - lib/polyn/cloud-event-schema.json
+- lib/polyn/templates/.dockerignore
+- lib/polyn/templates/.gitignore
+- lib/polyn/templates/Dockerfile
+- lib/polyn/templates/Gemfile
 - lib/polyn/templates/README.md
 - lib/polyn/templates/docker-compose.yml
 - lib/polyn/templates/events/.gitkeep
 - lib/polyn/templates/events/widgets.created.v1.json
 - lib/polyn/templates/generators/schema.json
 - lib/polyn/templates/generators/stream.tf
-- lib/polyn/templates/gitignore
-- lib/polyn/templates/tf/event_registry.tf
-- lib/polyn/templates/tf/main.tf
+- lib/polyn/templates/tf/kv_buckets.tf
+- lib/polyn/templates/tf/provider.tf
+- lib/polyn/templates/tf/remote_state_config/backend.tf
+- lib/polyn/templates/tf/variables.tf
+- lib/polyn/templates/tf/versions.tf
 - lib/polyn/templates/tf/widgets.tf
 - polyn-cli.gemspec
 homepage: https://github.com/Spiffinc/polyn-cli

data/lib/polyn/templates/gitignore

@@ -1,3 +0,0 @@
-tf/.terraform/**
-*.tfstate
-.terraform.lock.hcl

data/lib/polyn/templates/tf/main.tf

@@ -1,17 +0,0 @@
-terraform {
-  required_providers {
-    jetstream = {
-      source = "nats-io/jetstream"
-    }
-  }
-
-}
-
-variable "jetstream_servers" {
-  type = string
-  description = "The JetStream servers to connect to"
-}
-
-provider "jetstream" {
-  servers = var.jetstream_servers
-}