terraorg 0.1.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9dcc3e713e1ec4a3166adfd99ecb9cccb52efd37f489fc18ff2393be0b79ac1a
-  data.tar.gz: 0d12305ad0d7f2aa129336010f719f0d1361c0d28b2fa9466c2c1205d280830b
+  metadata.gz: 46107b71a1eace06c51463513c5b495b9549ec19ebc911103ec0d7f236fec6f8
+  data.tar.gz: b05708c67d359a3040eca8724140603d5c4debd6e2f1a25c87034e8a9b60e7be
 SHA512:
-  metadata.gz: 19df8bcc655abfe6eb89a7d5e8d50d7f76298a492b14cb2f37369b17b2f396822897d264d6f861f215faabd0e74af0dc72a23756cac1e319c34d36a60c915783
-  data.tar.gz: c3b93acbeaa7f18f6f64c4343944649acf2630a99823b53fce3789c20fb1fa8a5be8bebf9159c2a48ae39eb96b5d4af1c669b23d879087a38104f36d330aeb09
+  metadata.gz: 9f9286df25676340a5e3a36221f5b7de4ed21cce63281850210f18b360474544e453939b8b4559dec6cb58dfa0b9ce5facca21570f03295cb5f9d0b5c56eff57
+  data.tar.gz: 196d63d8df921c54216511ee7604b6ec8f8813241609a1fb0c4fa5480d965c4d2e9f0b2042a0f55a46b1fbaa8cb633f50317608afbf68f71d42b8b00fc877f83

data/README.md

@@ -34,7 +34,9 @@ Based on the org that this tool was originally designed for, orgs are expected
 to have three levels:
 
 * *squads*: the base unit of team-dom, containing people, who may be in
-  different geographical regions.
+  different geographical regions. Teams contain _members_ (full time heads)
+  and _associates_ (typically part time floaters.) Any associate of a squad
+  must also have a home squad for which they are a full time member.
 * *platoons*: a unit which contains squads and exceptional people who are
   members of the platoon, but not part of any squad
 * *org*: The whole organization, including its manager, any exceptional squads
@@ -45,6 +47,10 @@ The tool generates groups for each granular unit of organization in Okta and G
 Suite in Terraform. With patching, it could be possible for more organizational
 systems to be supported.
 
+## Diagram
+
+![Diagram of org structure](img/diagram.png)
+
 ## How it works
 
 Firstly, take your entire existing organization and define it using the
@@ -120,6 +126,10 @@ information on how to configure the providers.
 [articulate/terraform-provider-okta]: https://github.com/articulate/terraform-provider-okta
 [DeviaVir/terraform-provider-gsuite]: https://github.com/DeviaVir/terraform-provider-gsuite
 
+## Running tests
+There are a limited number of tests that can be invoked with 
+`ruby -I lib  test/terraorg/model/org_test.rb `
+
 ## Suggested process
 
 At [LiveRamp], a pull request based workflow leveraging [Atlantis] is used to

data/bin/terraorg

@@ -31,14 +31,15 @@ ACTIONS = [
   'validate'
 ].freeze
 
+STRICT_VALIDATION = ENV.fetch('TERRAORG_STRICT_VALIDATION', 'true')
 SQUADS_FILE = ENV.fetch('TERRAORG_SQUADS', 'squads.json')
 PLATOONS_FILE = ENV.fetch('TERRAORG_PLATOONS', 'platoons.json')
 ORG_FILE = ENV.fetch('TERRAORG_ROOT', 'org.json')
 CACHE_FILE = ENV.fetch('TERRAORG_OKTA_CACHE', 'okta_cache.json')
-GSUITE_DOMAIN = ENV.fetch('GSUITE_DOMAIN')
-SLACK_DOMAIN = ENV.fetch('SLACK_DOMAIN', 'yourcompany.slack.com')
-OKTA_ORG_NAME = ENV.fetch('OKTA_ORG_NAME')
-OKTA_API_TOKEN = ENV.fetch('OKTA_API_TOKEN')
+GSUITE_DOMAIN = ENV.fetch('GSUITE_DOMAIN', 'fillmein_gsuite.com')
+SLACK_DOMAIN = ENV.fetch('SLACK_DOMAIN', 'fillmein.slack.com')
+OKTA_ORG_NAME = ENV.fetch('OKTA_ORG_NAME', 'fillmein_okta')
+OKTA_API_TOKEN = ENV.fetch('OKTA_API_TOKEN', 'fillmein_okta_api_token')
 OKTA_BASE_URL = ENV.fetch('OKTA_BASE_URL', 'okta.com')
 
 action = ARGV[0]
@@ -95,7 +96,8 @@ platoons = Platoons.new(JSON.parse(platoons_data), squads, people, GSUITE_DOMAIN
 org_data = File.read(ORG_FILE)
 org = Org.new(JSON.parse(org_data), platoons, squads, people, GSUITE_DOMAIN)
 
-org.validate!
+strict = (STRICT_VALIDATION == 'true')
+org.validate!(strict: strict)
 
 case action
 when 'generate-squads-md'

data/lib/terraorg/model/org.rb

@@ -15,7 +15,8 @@
 require 'terraorg/model/util'
 
 class Org
-  MAX_SQUADS_PER_PERSON = 2
+  MAX_MEMBER_SQUADS_PER_PERSON = 1
+  MAX_ASSOCIATE_SQUADS_PER_PERSON = 3
   SCHEMA_VERSION = 'v1'.freeze
 
   def initialize(parsed_data, platoons, squads, people, gsuite_domain)
@@ -48,23 +49,33 @@ class Org
     @squads = squads
   end
 
-  def validate!
+  def validate!(strict: true)
+    failure = false
+
     # Do not allow the JSON files to contain any people who have left.
-    raise "Users have left the company: #{@people.inactive.map(&:id).join(', ')}" unless @people.inactive.empty?
+    unless @people.inactive.empty?
+      $stderr.puts "ERROR: Users have left the company, or are Suspended in Okta: #{@people.inactive.map(&:id).join(', ')}"
+      failure = true
+    end
 
     # Do not allow the org to be totally empty.
-    raise 'Org has no platoons or exception squads' if @member_platoons.size + @member_exception_squads.size == 0
+    if @member_platoons.size + @member_exception_squads.size == 0
+      $stderr.puts 'ERROR: Org has no platoons or exception squads'
+      failure = true
+    end
 
     # Require all platoons to be part of the org.
     platoon_diff = Set.new(@platoons.all_names) - Set.new(@member_platoon_names)
     unless platoon_diff.empty?
-      raise "Platoons are not used in the org: #{platoon_diff.to_a.sort}"
+      $stderr.puts "ERROR: Platoons are not used in the org: #{platoon_diff.to_a.sort}"
+      failure = true
     end
 
     # Require all squads to be used in the org.
     squad_diff = Set.new(@squads.all_names) - Set.new(@platoons.all_squad_names) - Set.new(@member_exception_squad_names)
     unless squad_diff.empty?
-      raise "Squad(s) are not used in the org: #{squad_diff.to_a.sort}"
+      $stderr.puts "ERROR: Squad(s) are not used in the org: #{squad_diff.to_a.sort}"
+      failure = true
     end
 
     all_squads = (@member_platoons.map(&:member_squads) + @member_exception_squads).flatten
@@ -78,20 +89,37 @@ class Org
       count > 1
     end
     if !more_than_one_platoon.empty?
-      raise "Squads are part of more than one platoon: #{more_than_one_platoon}"
+      $stderr.puts "ERROR: Squads are part of more than one platoon: #{more_than_one_platoon}"
+      failure = true
     end
 
-    # Validate that a squad member belongs to at most two squads in the entire org
+    # Validate that a squad member belongs to some maximum number of squads
+    # across the entire org. A person can be an associate of other squads
+    # at a different count. See top of file for defined limits.
     squad_count = {}
-    all_squads.map(&:teams).flatten.map(&:values).flatten.map(&:members).flatten.each do |member|
+    all_members = all_squads.map(&:teams).flatten.map(&:values).flatten.map(&:members).flatten
+    all_members.each do |member|
       squad_count[member.id] = squad_count.fetch(member.id, 0) + 1
     end
     more_than_max_squads = squad_count.select do |member, count|
-      count > MAX_SQUADS_PER_PERSON
+      count > MAX_MEMBER_SQUADS_PER_PERSON
     end
     if !more_than_max_squads.empty?
-      # TODO(joshk): Enforce after consulting with Sean
-      $stderr.puts "WARNING: Members are part of more than #{MAX_SQUADS_PER_PERSON} squads: #{more_than_max_squads}"
+      $stderr.puts "ERROR: People are members of more than #{MAX_MEMBER_SQUADS_PER_PERSON} squads: #{more_than_max_squads}"
+      failure = true
+    end
+
+    associate_count = {}
+    all_associates = all_squads.map(&:teams).flatten.map(&:values).flatten.map(&:associates).flatten
+    all_associates.each do |assoc|
+      associate_count[assoc.id] = associate_count.fetch(assoc.id, 0) + 1
+    end
+    more_than_max_squads = associate_count.select do |_, count|
+      count > MAX_ASSOCIATE_SQUADS_PER_PERSON
+    end
+    if !more_than_max_squads.empty?
+      $stderr.puts "ERROR: People are associates of more than #{MAX_ASSOCIATE_SQUADS_PER_PERSON} squads: #{more_than_max_squads}"
+      failure = true
     end
 
     # Validate that a squad member is not also an org exception
@@ -100,8 +128,18 @@ class Org
       exceptions.member? member
     end
     if !exception_and_squad_member.empty?
-      raise "Exception members are also squad members: #{exception_and_squad_member}"
+      $stderr.puts "ERROR: Exception members are also squad members: #{exception_and_squad_member}"
+      failure = true
+    end
+
+    # Validate that any associate is a member of some squad
+    associates_but_not_members = Set.new(all_associates.map(&:id)) - Set.new(all_members.map(&:id)) - exceptions
+    if !associates_but_not_members.empty?
+      $stderr.puts "ERROR: #{associates_but_not_members.map(&:id)} are associates of squads but not members of any squad"
+      failure = true
     end
+
+    raise "CRITICAL: Validation failed due to at least one error above" if failure && strict
   end
 
   def members
@@ -153,8 +191,8 @@ class Org
     md_lines = [
       '# Engineering Squads List',
       '',
-      '|Platoon|Squad|PM|Mailing list|TS SME|Slack|# Engineers|Squad Manager|Eng Product Owner|Members|',
-      '|---|---|---|---|---|---|---|---|---|---|',
+      '|Platoon|Squad|PM|Mailing list|TS SME|Slack|# Engineers|Squad Manager|Members|',
+      '|---|---|---|---|---|---|---|---|---|',
     ]
     md_lines += @member_platoons.map { |s| s.get_squads_psv_rows(@id) }
     md_lines += @member_exception_squads.map { |s| s.to_md('_No Platoon_', @id) }
@@ -164,13 +202,16 @@ class Org
     md_lines.join("\n")
   end
 
-  def generate_tf
-    tf = @member_platoons.map { |p| p.generate_tf(@id) }.join("\n")
-    File.write('auto.platoons.tf', tf)
+  def generate_tf_platoons
+    @member_platoons.map { |p| p.generate_tf(@id) }.join("\n")
+  end
 
-    tf = @member_exception_squads.map { |s| s.generate_tf(@id) }.join("\n")
-    File.write('auto.exception_squads.tf', tf)
+  def generate_tf_squads
+    @member_exception_squads.map { |s| s.generate_tf(@id) }.join("\n")
+  end
 
+  def generate_tf_org
+    tf = ''
     # Roll all platoons and exception squads into the org.
     roll_up_to_org = \
       @member_exception_squads.map { |s| s.unique_name(@id, nil) } + \
@@ -210,14 +251,18 @@ EOF
     all_locations[@manager_location] = all_locations.fetch(@manager_location, Set.new).add(@manager)
 
     all_locations.each do |l, m|
+      description = "#{@name} organization members based in #{l} (terraorg)"
       name = "#{unique_name}-#{l.downcase}"
       tf += <<-EOF
 resource "okta_group" "#{name}" {
   name = "#{name}"
-  description = "#{@name} organization members based in #{l} (terraorg)"
+  description = "#{description}"
   users = #{Util.persons_tf(m)}
 }
+
+#{Util.gsuite_group_tf(name, @gsuite_domain, m, description)}
 EOF
+
     end
 
     # Generate a special GSuite group for all managers (org, platoon, squad
@@ -226,7 +271,17 @@ EOF
     all_managers = Set.new([@manager] + @platoons.all.map(&:manager) + @squads.all.map(&:manager).select { |m| m })
     manager_dl = "#{@id}-managers"
     tf += Util.gsuite_group_tf(manager_dl, @gsuite_domain, all_managers, "All managers of the #{@name} organization (terraorg)")
+    tf
+  end
+
+  def generate_tf
+    tf = generate_tf_platoons
+    File.write('auto.platoons.tf', tf)
+
+    tf = generate_tf_squads
+    File.write('auto.exception_squads.tf', tf)
 
+    tf = generate_tf_org
     File.write('auto.org.tf', tf)
   end
 

data/lib/terraorg/model/platoon.rb

@@ -99,9 +99,9 @@ EOF
   # - Sort the squad ids lexically
   # - Sort the exceptions lexically
   def to_h
-    obj = { 'id' => @id, 'name' => @name, 'manager' => @manager.id, 'squads' => @member_squads.map(&:id) }
+    obj = { 'id' => @id, 'name' => @name, 'manager' => @manager.id, 'squads' => @member_squads.map(&:id).sort }
     unless @member_exceptions.empty?
-      obj['exceptions'] = @member_exceptions.map(&:id)
+      obj['exceptions'] = @member_exceptions.map(&:id).sort
     end
     unless @metadata.empty?
       obj['metadata'] = @metadata

data/lib/terraorg/model/squad.rb

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'countries'
+
 require 'terraorg/model/people'
 require 'terraorg/model/util'
 
@@ -19,23 +21,49 @@ class Squad
   attr_accessor :id, :name, :metadata, :teams
 
   class Team
-    attr_accessor :location, :members
+    attr_accessor :location, :members, :associates
 
     def initialize(parsed_data, people)
-      @location = parsed_data.fetch('location')
+      location = parsed_data.fetch('location')
+      country = ISO3166::Country.new(location)
+      raise "Location is invalid: #{location}" unless country
+      @location = country.alpha2
       @members = parsed_data.fetch('members', []).map do |n|
         people.get_or_create!(n)
       end
+      @associates = parsed_data.fetch('associates', []).map do |n|
+        people.get_or_create!(n)
+      end
+    end
+
+    def validate!
+      raise 'Subteam has no full time members' if @members.size == 0
+      # location validation done at initialize time
+      # associates can be empty
+
+      # associates and members must have zero intersection
+      associate_set = Set.new(@associates.map(&:id))
+      member_set = Set.new(@members.map(&:id))
+      raise 'A member cannot also be an associate of the same team' if associate_set.intersection(member_set)
     end
 
     # Output a canonical (sorted, formatted) version of this Team.
     # - Sort the members in each team
+    # - Only add an associates field if it's present
     def to_h
-      { 'location' => @location, 'members' => @members.map(&:id).sort }
+      {
+        'associates' => @associates.map(&:id).sort,
+        'location' => @location,
+        'members' => @members.map(&:id).sort,
+      }
+    end
+
+    def everyone
+      @associates + @members
     end
 
     def to_md
-      "**#{@location}**: #{@members.map(&:name).sort.join(', ')}"
+      "**#{@location}**: #{@members.map(&:name).sort.join(', ')}, #{@associates.map { |m| "_#{m.name}_" }.sort.join(', ')}"
     end
   end
 
@@ -53,10 +81,18 @@ class Squad
     @teams = Hash[teams_arr.map { |t| [t.location, t] }]
   end
 
-  def members(location: nil)
+  # Everyone including associates on all subteams in the squad.
+  def everyone(location: nil)
     @teams.select { |l, t|
       location == nil || l == location
-    }.map { |l, t|
+    }.map { |_, t|
+      t.everyone
+    }.flatten
+  end
+
+  # Full-time members of all subteams in this squad
+  def members
+    @teams.map { |_, t|
       t.members
     }.flatten
   end
@@ -64,11 +100,11 @@ class Squad
   def get_acl_groups(org_id)
     # each geographically located subteam
     groups = Hash[@teams.map { |location, team|
-      [unique_name(org_id, location), {'name' => "#{@name} squad members based in #{location}", 'members' => team.members}]
+      [unique_name(org_id, location), {'name' => "#{@name} squad members based in #{location}", 'members' => team.everyone}]
     }]
 
     # combination of all subteams
-    groups[unique_name(org_id, nil)] = {'name' => "#{@name} squad worldwide members", 'members' => members}
+    groups[unique_name(org_id, nil)] = {'name' => "#{@name} squad worldwide members", 'members' => everyone}
 
     groups
   end
@@ -82,7 +118,7 @@ class Squad
   end
 
   def validate!
-    raise 'Squad has no members' if members.size == 0
+    @teams.each(&:validate!)
   end
 
   def to_md(platoon_name, org_id)
@@ -94,11 +130,6 @@ class Squad
       sme = @people.get_or_create!(sme).name
     end
 
-    epo = @metadata.fetch('epo', '')
-    if !epo.empty?
-      epo = @people.get_or_create!(epo).name
-    end
-
     manager = @metadata.fetch('manager', '')
     if !manager.empty?
       manager = @people.get_or_create!(manager).name
@@ -110,8 +141,8 @@ class Squad
     if slack
       slack = "[#{slack}](https://#{@slack_domain}/app_redirect?channel=#{slack.gsub(/^#/, '')})"
     end
-    # platoon name, squad name, PM, email list, SME, slack, # people, squad manager, eng product owner, members
-    "|#{platoon_name}|#{@name}|#{pm}|[#{email}](#{email})|#{sme}|#{slack}|#{members.size}|#{manager}|#{epo}|#{subteam_members}|"
+    # platoon name, squad name, PM, email list, SME, slack, # full time members, squad manager, members
+    "|#{platoon_name}|#{@name}|#{pm}|[#{email}](#{email})|#{sme}|#{slack}|#{members.size}|#{manager}|#{subteam_members}|"
   end
 
   def generate_tf(org_id)

data/lib/terraorg/version.rb

@@ -13,5 +13,5 @@
 # limitations under the License.
 
 module Terraorg
-  VERSION = '0.1.0'
+  VERSION = '0.5.0'
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: terraorg
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Joshua Kwan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-30 00:00:00.000000000 Z
+date: 2020-10-12 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: countries
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.14'
 description: Manage an organizational structure with Okta and G-Suite using Terraform
 email: joshk@triplehelix.org
 executables:
@@ -90,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: terraorg