polyn-cli 0.1.0

data/lib/polyn/cli/schema_loader.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Polyn
+  class Cli
+    ##
+    # Loads the JSON schmea into the event registry.
+    class SchemaLoader
+      include Thor::Actions
+
+      STORE_NAME = "POLYN_SCHEMAS"
+
+      ##
+      # Loads the events from the event repository into the Polyn event registry.
+      # @return [Bool]
+      def self.load(cli)
+        new(cli).load_events
+      end
+
+      def initialize(thor, **opts)
+        @thor               = thor
+        @client             = NATS.connect(Polyn::Cli.configuration.nats_servers).jetstream
+        @bucket             = client.key_value(opts.fetch(:store_name, STORE_NAME))
+        @cloud_event_schema = Polyn::Cli::CloudEvent.to_h.freeze
+        @events_dir         = opts.fetch(:events_dir, File.join(Dir.pwd, "events"))
+        @events             = {}
+      end
+
+      def load_events
+        thor.say "Loading events into the Polyn event registry from '#{events_dir}'"
+        read_events
+
+        events.each do |name, event|
+          bucket.put(name, JSON.generate(event))
+        end
+
+        true
+      end
+
+      private
+
+      attr_reader :thor, :events, :client, :bucket, :cloud_event_schema, :events_dir
+
+      def read_events
+        Dir.glob(File.join(events_dir, "*.json")).each do |event_file|
+          thor.say "Loading 'event #{event_file}'"
+          data_schema = JSON.parse(File.read(event_file))
+          event_type  = File.basename(event_file, ".json")
+          validate_schema!(event_type, data_schema)
+          Polyn::Cli::Naming.validate_event_type!(event_type)
+          schema      = compose_cloud_event(data_schema)
+
+          events[event_type] = schema
+        end
+      end
+
+      def validate_schema!(event_type, schema)
+        JSONSchemer.schema(schema)
+      rescue StandardError => e
+        raise Polyn::Cli::ValidationError,
+          "Invalid JSON Schema document for event #{event_type}\n#{e.message}\n"\
+          "#{JSON.pretty_generate(schema)}"
+      end
+
+      def compose_cloud_event(event_schema)
+        cloud_event_schema.merge({
+          "definitions" => cloud_event_schema["definitions"].merge({
+            "datadef" => event_schema,
+          }),
+        })
+      end
+    end
+  end
+end

data/lib/polyn/cli/stream_generator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Polyn
+  class Cli
+    ##
+    # Generates a new Stream configuration file for terraform
+    class StreamGenerator < Thor::Group
+      include Thor::Actions
+
+      desc "Generates a new stream configuration with boilerplate"
+
+      argument :name, required: true
+      class_option :dir, default: Dir.getwd
+
+      source_root File.join(File.expand_path(__dir__), "../templates")
+
+      def check_name
+        Polyn::Cli::Naming.validate_stream_name!(name)
+      end
+
+      def file_name
+        @file_name ||= name.downcase
+      end
+
+      def stream_name
+        @stream_name ||= Polyn::Cli::Naming.format_stream_name(name)
+      end
+
+      def create
+        say "Creating new stream config #{stream_name}"
+        template "generators/stream.tf", File.join(options.dir, "tf/#{file_name}.tf")
+      end
+    end
+  end
+end

data/lib/polyn/cli/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Polyn
+  class Cli
+    VERSION = "0.1.0"
+  end
+end

data/lib/polyn/cli.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require "thor"
+require "dotenv"
+require "json_schemer"
+require "polyn/cli/configuration"
+require "polyn/cli/consumer_generator"
+require "polyn/cli/naming"
+require "polyn/cli/schema_generator"
+require "polyn/cli/stream_generator"
+require "polyn/cli/cloud_event"
+require "polyn/cli/schema_loader"
+require "polyn/cli/version"
+require "json"
+require "nats/client"
+
+Dotenv.load
+
+module Polyn
+  ##
+  # CLI for Polyn for configuring NATS server
+  class Cli
+    ##
+    # Proxy to Thor start
+    def self.start(args)
+      Commands.start(args)
+    end
+
+    class Error < StandardError; end
+    class ValidationError < Error; end
+
+    ##
+    # Configuration information for Polyn
+    def self.configuration
+      @configuration ||= Polyn::Cli::Configuration.new
+    end
+
+    ##
+    # Thor commands for the CLI. Subclassed so other classes can be in the CLI namespace
+    class Commands < Thor
+      include Thor::Actions
+
+      source_root File.join(File.expand_path(__dir__), "templates")
+
+      # https://github.com/rails/thor/wiki/Making-An-Executable
+      def self.exit_on_failure?
+        true
+      end
+
+      method_option :dir, default: Dir.getwd
+      desc "init", "initializes a Polyn event repository"
+      def init
+        say "Initializing Polyn event repository"
+        directory "tf", File.join(options.dir, "tf")
+        directory "events", File.join(options.dir, "events")
+        template "docker-compose.yml", File.join(options.dir, "docker-compose.yml")
+        template "gitignore", File.join(options.dir, ".gitignore")
+        template "README.md", File.join(options.dir, "README.md")
+        run tf_init
+        say "Initializing git"
+        inside options.dir do
+          run "git init"
+        end
+        say "Repository initialized"
+      end
+
+      desc "tf_init", "Initializes Terraform for configuration"
+      def tf_init
+        say "Initializing Terraform"
+        inside File.join(options.dir, "tf") do
+          run "terraform init"
+        end
+      end
+
+      desc "up", "updates the JetStream streams and consumers, as well the Polyn event registry"
+      def up
+        say "Updating JetStream configuration"
+        inside "tf" do
+          run tf_apply
+        end
+        say "Updating Polyn event registry"
+        Polyn::Cli::SchemaLoader.new(self).load_events
+      end
+
+      private
+
+      def polyn_env
+        Polyn::Cli.configuration.polyn_env
+      end
+
+      def nats_servers
+        Polyn::Cli.configuration.nats_servers
+      end
+
+      def tf_apply
+        if polyn_env == "development"
+          %(terraform apply -var "jetstream_servers=#{nats_servers}" -auto-approve)
+        else
+          %(terraform apply -var "jetstream_servers=#{nats_servers}")
+        end
+      end
+
+      register(Polyn::Cli::SchemaGenerator, "gen:schema", "gen:schema EVENT_TYPE",
+        "Generates a new JSON Schema file for an event")
+      register(Polyn::Cli::StreamGenerator, "gen:stream", "gen:stream NAME",
+        "Generates a new stream configuration with boilerplate")
+      register(Polyn::Cli::ConsumerGenerator, "gen:consumer",
+        "gen:consumer STREAM_NAME DESTINATION_NAME EVENT_TYPE",
+        "Generates a new NATS Consumer configuration with boilerplate")
+    end
+  end
+end

data/lib/polyn/cloud-event-schema.json

@@ -0,0 +1,187 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "CloudEvents Specification JSON Schema, extended for Polyn",
+  "type": "object",
+  "properties": {
+    "id": {
+      "description": "Identifies the event.",
+      "$ref": "#/definitions/iddef",
+      "examples": [
+        "A234-1234-1234"
+      ]
+    },
+    "source": {
+      "description": "Identifies the context in which an event happened.",
+      "$ref": "#/definitions/sourcedef",
+      "examples" : [
+        "https://github.com/cloudevents",
+        "mailto:cncf-wg-serverless@lists.cncf.io",
+        "urn:uuid:6e8bc430-9c3a-11d9-9669-0800200c9a66",
+        "cloudevents/spec/pull/123",
+        "/sensors/tn-1234567/alerts",
+        "1-555-123-4567"
+      ]
+    },
+    "specversion": {
+      "description": "The version of the CloudEvents specification which the event uses.",
+      "$ref": "#/definitions/specversiondef",
+      "examples": [
+        "1.0"
+      ]
+    },
+    "type": {
+      "description": "Describes the type of event related to the originating occurrence.",
+      "$ref": "#/definitions/typedef",
+      "examples" : [
+        "com.github.pull_request.opened",
+        "com.example.object.deleted.v2"
+      ]
+    },
+    "datacontenttype": {
+      "description": "Content type of the data value. Must adhere to RFC 2046 format.",
+      "$ref": "#/definitions/datacontenttypedef",
+      "examples": [
+        "text/xml",
+        "application/json",
+        "image/png",
+        "multipart/form-data"
+      ]
+    },
+    "dataschema": {
+      "description": "Identifies the schema that data adheres to.",
+      "$ref": "#/definitions/dataschemadef"
+    },
+    "subject": {
+      "description": "Describes the subject of the event in the context of the event producer (identified by source).",
+      "$ref": "#/definitions/subjectdef",
+      "examples": [
+        "mynewfile.jpg"
+      ]
+    },
+    "time": {
+      "description": "Timestamp of when the occurrence happened. Must adhere to RFC 3339.",
+      "$ref": "#/definitions/timedef",
+      "examples": [
+        "2018-04-05T17:31:00Z"
+      ]
+    },
+    "data": {
+      "description": "The event payload.",
+      "$ref": "#/definitions/datadef",
+      "examples": [
+        "<much wow=\"xml\"/>"
+      ]
+    },
+    "data_base64": {
+      "description": "Base64 encoded event payload. Must adhere to RFC4648.",
+      "$ref": "#/definitions/data_base64def",
+      "examples": [
+        "Zm9vYg=="
+      ]
+    },
+    "polyndata": {
+      "$ref": "#/definitions/polyndatadef",
+      "description": "Information about the client that produced the event and additional metadata",
+      "examples": [
+        {
+          "clientlang": "elixir",
+          "clientlangversion": "1.13.2",
+          "clientversion": "0.1.0"
+        }
+      ]
+    },
+    "polyntrace": {
+      "$ref": "#/definitions/polyntracedef",
+      "description": "Previous events that led to this one",
+      "examples": [
+        [
+          {
+            "type": "<topic>",
+            "time": "2018-04-05T17:31:00Z",
+            "id": "<uuid>"
+          }
+        ]
+      ]
+    }
+  },
+  "required": ["id", "source", "specversion", "type"],
+  "definitions": {
+    "iddef": {
+      "type": "string",
+      "minLength": 1
+    },
+    "sourcedef": {
+      "type": "string",
+      "format": "uri-reference",
+      "minLength": 1
+    },
+    "specversiondef": {
+      "type": "string",
+      "minLength": 1
+    },
+    "typedef": {
+      "type": "string",
+      "minLength": 1
+    },
+    "datacontenttypedef": {
+      "type": ["string", "null"],
+      "minLength": 1
+    },
+    "dataschemadef": {
+      "type": ["string", "null"],
+      "format": "uri",
+      "minLength": 1
+    },
+    "subjectdef": {
+      "type": ["string", "null"],
+      "minLength": 1
+    },
+    "timedef": {
+      "type": ["string", "null"],
+      "format": "date-time",
+      "minLength": 1
+    },
+    "datadef": {
+      "type": ["object", "string", "number", "array", "boolean", "null"]
+    },
+    "data_base64def": {
+      "type": ["string", "null"],
+      "contentEncoding": "base64"
+    },
+    "polyndatadef": {
+      "type": "object",
+      "properties": {
+        "clientlang": {
+          "type": "string"
+        },
+        "clientlangversion": {
+          "type": "string"
+        },
+        "clientversion": {
+          "type": "string"
+        }
+      },
+      "required": ["clientlang", "clientlangversion", "clientversion"]
+    },
+    "polyntracedef": {
+      "type" : "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "type": {
+            "type": "string"
+          },
+          "time": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "id"  : {
+            "type": "string",
+            "format": "uuid"
+          }
+        },
+        "required": ["type", "time", "id"]
+      }
+    }
+  }
+}

data/lib/polyn/templates/README.md

@@ -0,0 +1,51 @@
+# Polyn Events Repository
+
+This repository contains all of the events and terraform resources for the Polyn services
+environment.
+
+## Development Setup
+
+1. Install [Ruby](https://github.com/asdf-vm/asdf-ruby)
+2. Install Terraform. For M1 Macs, [download the AMD64 version](https://www.terraform.io/downloads)
+   rather than using Homebrew to install Terraform.
+3. Ensure Docker & Docker Compose is installed
+4. [Install the Polyn CLI]()
+5. Call `polyn up`. By default this will run in `development` mode, which will start the NATS
+   server, configure it via Terraform, and update the Polyn Event Registry.
+
+## Streams
+
+Each stream should have its own configuration file under `./tf` . Run `polyn gen:stream <stream_name>` to generate a new configuration file for a stream
+
+## Consumers
+
+Run `polyn gen:consumer <stream_name> <destination_name> <event_type>` to generate new configuration for a consumer of a stream. It will be included in the same file as the stream configuration.
+
+## Event Schemas
+
+Run `polyn gen:schema <event_type>` to generate a new JSON Schema for an event
+
+All the schemas for your events should live in the `./events` directory.
+The name of your schema file should be the same as your event, but with `.json` at the end.
+So if you have an event called `widgets.created.v1` you would create a schema file called `widgets.created.v1.json` in the `./events` directory.
+Every schema should be a valid [JSON Schema](https://json-schema.org/) document.
+The Polyn CLI tool will combine your event schema with the [Cloud Events Schema](https://cloudevents.io/) when it adds it to the Polyn Event Registry.
+This means you only need to include the JSON Schema for the `data` portion of the Cloud Event and not the entire Cloud Event schema.
+
+## Schema Versioning
+
+### New Event
+
+A new event schema file should be a lower-case, dot-separated, name with a `v1` suffix
+
+### Existing Event
+
+Existing event schemas can be changed without updating the file name if the change is backwards-compatible.
+Backwards-compatibile meaning that any services Producing or Consuming the event will not break or be invalid when the
+Polyn Event Registry is updated with the change. There are many ways to make breaking change and so you should be
+careful when you do this.
+
+Making a change to an event schema that is not backwards-compatible will require you to create a brand new
+json file. The new file should have the same name as your old file, but with the version number increased. Your
+Producers will need to continue producing both events until you are sure there are no more consumers using the
+old event.

data/lib/polyn/templates/docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3"
+
+services:
+  nats:
+    image: nats:2.8.4
+    ports:
+      - "4222:4222"
+    command:
+      - -js

data/lib/polyn/templates/events/widgets.created.v1.json

@@ -0,0 +1,14 @@
+{
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": "string",
+      "description": "The id of the widget"
+    },
+    "name": {
+      "type": "string",
+      "description": "The name of the widget"
+    }
+  },
+  "required": ["id", "name"]
+}

data/lib/polyn/templates/generators/schema.json

@@ -0,0 +1,9 @@
+{
+  "$id": "<%= schema_id %>",
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "description": "This is why this event exists and what it does",
+  "type": "object",
+  "properties": {
+
+  }
+}

data/lib/polyn/templates/generators/stream.tf

@@ -0,0 +1,10 @@
+// STREAM DEFINITION
+
+resource "jetstream_stream" "<%= stream_name %>" {
+  name = "<%= stream_name %>"
+  subjects = []
+  storage = "file"
+  max_age  = 60 * 60 * 24 * 365 // 1 year
+}
+
+// CONSUMERS

data/lib/polyn/templates/gitignore

@@ -0,0 +1,3 @@
+tf/.terraform/**
+*.tfstate
+.terraform.lock.hcl

data/lib/polyn/templates/tf/event_registry.tf

@@ -0,0 +1,3 @@
+resource "jetstream_kv_bucket" "POLYN_SCHEMAS" {
+  name = "POLYN_SCHEMAS"
+}

data/lib/polyn/templates/tf/main.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    jetstream = {
+      source = "nats-io/jetstream"
+    }
+  }
+
+}
+
+variable "jetstream_servers" {
+  type = string
+  description = "The JetStream servers to connect to"
+}
+
+provider "jetstream" {
+  servers = var.jetstream_servers
+}

data/lib/polyn/templates/tf/widgets.tf

@@ -0,0 +1,18 @@
+// STREAM DEFINITION
+
+resource "jetstream_stream" "WIDGETS" {
+  name = "WIDGETS"
+  subjects = ["widgets.>"]
+  storage = "file"
+  max_age  = 60 * 60 * 24 * 365 // 1 year
+}
+
+// CONSUMERS
+
+resource "jetstream_consumer" "created_notifier_widgets_created_v1" {
+  stream_id = jetstream_stream.WIDGETS.id
+  durable_name = "created_notifier_widgets_created_v1"
+  deliver_all = true
+  filter_subject = "widgets.created.v1"
+  sample_freq = 100
+}

data/polyn-cli.gemspec

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "lib/polyn/cli/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "polyn-cli"
+  spec.version       = Polyn::Cli::VERSION
+  spec.authors       = ["Jarod", "Brandyn Bennett"]
+  spec.email         = ["jarod.reid@spiff.com", "brandyn.bennett@spiff.com"]
+
+  spec.summary               = "CLI for the Polyn service framework"
+  spec.description           = "CLI for the Polyn service framework"
+  spec.homepage              = "https://github.com/Spiffinc/polyn-cli"
+  spec.license               = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "dotenv",    "~> 2.7.6"
+  spec.add_dependency "json_schemer", "~> 0.2"
+  spec.add_dependency "nats-pure", "~> 2.0.0"
+  spec.add_dependency "thor",      "~> 1.2.0"
+end