driftwood 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c22c3ad6165aea9d79243d7b720f931ca538b298
+  data.tar.gz: 1745c96c0da424c36cb06fb9a40c6b553569269a
+SHA512:
+  metadata.gz: 342f82ed397346348e5578d31c45ef6b8c8d47f11de7756253f3f04de58de9c70275f44671fcd4280ff848a523f4b769a30b91dcf3dd05fed8689e56af80168f
+  data.tar.gz: cdb16d92ad679d4e040feda4be6c486727f6c57e2a0f07caad2030fe28eeb34fa2003ba5dda378655421426b523da3ef3e205f96903bb63dc0b7f4cc0ddf699f

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# v0.0.1
+
+* Initial release.

data/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,122 @@
+# Driftwood
+Simple Slack logging and community management
+
+## Overview
+
+The Slack logging service isn't very complete, especially if you're on the free
+tier. For example, messages expire far too soon on an active channel and users
+don't include creation times. This tool provides those missing features, and soon
+will also provide features for building a positive team culture.
+
+## Operation
+
+At its core, the core functionality logs teams, channels, users, and messages to
+BigQuery. When you start the service for the first time, it starts a background
+thread to crawl the API and synchronize as much data as it can get or reconstruct.
+
+1. All channels in a team are iterated and recorded
+1. All users in a channel are iterated and recorded. If the user hasn't been seen
+   before, we assume that they were created while the service was stopped or during
+   restart and so record a creation time of now().
+1. All available messages that are newer than the newest message in our database
+   are iterated and recorded. This doesn't guarantee log consistency because
+   messages expire quickly and may be lost. Messages also do not have a unique ID,
+   though the microsecond timestamp is pretty close to it.
+
+Then a webhook & bot are started up. The webhook receives all events we've
+subscribed to and the bot is available for two-way conversations. Out of the box
+we just log messages and respond to a few silly messages, but that's easily
+extensible.
+
+   
+## Configuration
+
+Driftwood loads configuration from the first found of either `~/.driftwood/config.yaml`
+or `/etc/abalone/config.yaml`. You can pass the path to another config file at
+the command line. 
+
+Example configuration:
+
+```
+---
+:slack:
+  :client_id: "<client ID from Slack>"
+  :client_secret: <client secret from Slack>
+  :verification_token: <verification token from Slack>
+  :redirect_uri:  https://driftwood.example.com
+:gcloud:
+  :dataset: <dataset>
+  :project: <project>
+  :keyfile: ~/.driftwood/credentials.json
+```
+
+### Setting up integration
+
+The installation steps from https://github.com/slackapi/Slack-Ruby-Onboarding-Tutorial
+can be followed with a few obvious modifications. Configure and application for
+your team, and configure the service before starting it up.
+
+Once it's running, load up the webpage and press the **Add to Slack** button
+to add it to your workspace.
+
+
+## Extending
+
+Eventually this will have a real plugin system that makes this more obvious. For
+the time being, you'll have to invoke the server yourself and register any
+handlers you need.
+
+A simple echo bot:
+
+```
+require 'driftwood'
+
+options = {...}
+server  = Driftwood.new(options)
+server.slack.register_handler do |team_id, event_data|
+  next unless event_data['channel_type'] == 'im'
+  
+  user = event_data['user']
+  text = event_data['text']
+  name = server.slack.real_name(team_id, user)
+  
+  case text
+  when /^hello/i
+    server.slack.send_response(team_id, user, "Hi there, #{name}!")
+  when /^ping/i
+    server.slack.send_response(team_id, user, "Pong")
+  when /^echo/i
+    server.slack.send_response(team_id, user, "Echo response: #{text.sub(/^echo /, '')}")
+  end
+end
+
+server.run!
+```
+
+## Roadmap & Todo
+
+The next features coming up are those for community management and culture building.
+We'll be adding a templated onboarding process that will welcome people with a
+Getting Started guide that encourages the culture we want to develop and lays the
+foundation for community members to help self moderate and encourage the behaviours
+we'd like to see more of.
+Then we'll be adding multi-faceted karma tracking that allows members to reward
+each other for contributions (such as answering a question) and behaviours (such
+as kindness while welcoming a new member).
+Then we'll start doing language analysis so that we can automate more of this. For
+example, we'll look for certain keywords, and see who asks questions of who.
+ 
+
+## Limitations
+
+This is super early in development and has not yet been battle tested.
+
+## Disclaimer
+
+I take no liability for the use of this tool.
+
+Contact
+-------
+
+binford2k@gmail.com
+

data/bin/driftwood

@@ -0,0 +1,107 @@
+#! /usr/bin/env ruby
+
+require 'rubygems'
+require 'optparse'
+require 'yaml'
+require 'driftwood'
+
+defaults = {
+  :port    => 9292,
+  :bind    => '0.0.0.0',
+  :logfile => $stderr,
+  :slack   => {
+    :name  => 'driftwood',
+  },
+}
+loglevel   = Logger::WARN
+configfile = [ File.expand_path('~/.driftwood/config.yaml'), '/etc/driftwood/config.yaml'].select { |file| File.exist? file }.first
+options    = {}
+ssl_opts   = {:verify_peer => false}
+
+optparse = OptionParser.new { |opts|
+    opts.banner = "Usage : driftwood [-c config] [-p <port>] [-l [logfile]] [-d]
+         -- Runs the Driftwood slack community interaction service.
+
+"
+
+    opts.on("-c CONFIGFILE", "--config CONFIGFILE", "Load configuration from a file. (/etc/driftwood/config.yaml)") do
+        configfile = arg
+    end
+
+    opts.on("-d", "--debug", "Display or log debugging messages") do
+        loglevel = Logger::DEBUG
+    end
+
+    opts.on("-l [LOGFILE]", "--logfile [LOGFILE]", "Path to logfile. Defaults to no logging, or /var/log/driftwood if no filename is passed.") do |arg|
+        options[:logfile] = arg || '/var/log/driftwood'
+    end
+
+    opts.on("-p PORT", "--port PORT", "Port for the Slack webhook server to listen on. Defaults to 9292.") do |arg|
+        options[:port] = arg
+    end
+
+    opts.separator('')
+
+    opts.on("--ssl", "Run with SSL support. Autogenerates a self-signed certificates by default.") do
+        options[:ssl] = true
+    end
+
+    opts.on("--ssl-cert FILE", "Specify the SSL certificate you'd like use use. Pair with --ssl-key.") do |arg|
+        ssl_opts[:cert_chain_file] = arg
+    end
+
+    opts.on("--ssl-key FILE", "Specify the SSL key file you'd like use use. Pair with --ssl-cert.") do |arg|
+        ssl_opts[:private_key_file] = arg
+    end
+
+    opts.separator('')
+
+    opts.on("-h", "--help", "Displays this help") do
+        puts
+        puts opts
+        puts
+        exit
+    end
+}
+optparse.parse!
+
+config           = YAML.load_file(configfile) rescue {}
+config[:slack] ||= {}
+options[:slack]  = defaults[:slack].merge(config[:slack])
+options          = defaults.merge(config.merge(options))
+
+$logger       = Logger.new(options[:logfile])
+$logger.level = loglevel
+
+if ssl_opts[:cert_chain_file] and ssl_opts[:private_key_file]
+  options[:ssl] = true
+end
+
+# These options should either both be nil or both be Strings
+unless ssl_opts[:cert_chain_file].class == ssl_opts[:private_key_file].class
+  raise 'You must specify both the certificate and key file!'
+end
+
+if ARGV.first == 'shell'
+  require 'pry'
+  binding.pry
+  exit 0
+end
+
+puts
+puts
+puts "Starting Driftwood slack logbot. Browse to http://localhost:#{options[:port]}"
+puts
+puts
+
+Driftwood.run!(options) do |server|
+  if options[:ssl]
+    if server.respond_to? 'ssl='
+      $logger.info 'Enabling SSL support.'
+      server.ssl         = true
+      server.ssl_options = ssl_opts
+    else
+      $logger.warn "Please 'gem install thin' or run via an app server for SSL support."
+    end
+  end
+end

data/lib/driftwood/bigquery.rb

@@ -0,0 +1,154 @@
+require "google/cloud/bigquery"
+
+class Driftwood::Bigquery
+  def initialize(config)
+    $logger.info "Starting Bigquery connection"
+
+    config[:keyfile] = File.expand_path(config[:keyfile])
+
+    @bigquery = Google::Cloud::Bigquery.new(
+      :project_id  => config[:project],
+      :credentials => Google::Cloud::Bigquery::Credentials.new(config[:keyfile]),
+    )
+    @dataset = @bigquery.dataset(config[:dataset])
+    raise "\nThere is a problem with the gCloud configuration: \n #{JSON.pretty_generate(config)}" if @dataset.nil?
+
+    @teams = @dataset.table('slack_teams') || @dataset.create_table('slack_teams') do |table|
+                                                      table.name        = 'Slack Teams'
+                                                      table.description = 'A list of all team names & ID, plus some metadata'
+                                                      table.schema do |s|
+                                                        s.string "team_id",           mode: :required
+                                                        s.string "name",              mode: :required
+                                                        s.string "user_access_token", mode: :required
+                                                        s.string "bot_user_id",       mode: :required
+                                                        s.string "bot_access_token",  mode: :required
+                                                      end
+                                                    end
+
+    @creation = @dataset.table('slack_user_creation') || @dataset.create_table('slack_user_creation') do |table|
+                                                      table.name        = 'Slack User Creation'
+                                                      table.description = 'This keeps track of when users are created. The Slack API does not provide this info.'
+                                                      table.schema do |s|
+                                                        s.string    "team_id",   mode: :required
+                                                        s.string    "user_id",   mode: :required
+                                                        s.timestamp "creation",  mode: :required
+                                                      end
+                                                    end
+
+    @messages = @dataset.table('slack_messages') || @dataset.create_table('slack_messages') do |table|
+                                                      table.name        = 'Slack Message Log'
+                                                      table.description = 'All messages are logged here permanently.'
+                                                      table.schema do |s|
+                                                        s.string "team_id",    mode: :required
+                                                        s.string "channel_id", mode: :required
+                                                        s.string "user_id",    mode: :required
+                                                        s.string "ts",         mode: :required
+                                                        s.string "text",       mode: :required
+                                                      end
+                                                    end
+
+    # these we delete and rebuild because it's about 17 million times faster. Only slightly exaggerated.
+    @dataset.table('slack_channels').delete rescue nil
+    @channels = @dataset.create_table('slack_channels') do |table|
+      table.name        = 'Slack Channels'
+      table.description = 'A list of all channels and metadata'
+      table.schema do |s|
+        s.string  "team_id",     mode: :required
+        s.string  "channel_id",  mode: :required
+        s.string  "name",        mode: :required
+        s.integer "created",     mode: :required
+        s.boolean "is_private",  mode: :required
+        s.string  "topic",       mode: :required
+        s.string  "purpose",     mode: :required
+        s.integer "num_members", mode: :required
+      end
+    end
+
+    @dataset.table('slack_users').delete rescue nil
+    @users = @dataset.create_table('slack_users') do |table|
+      table.name        = 'Slack Users'
+      table.description = 'A list of all users and metadata. Ironically, we only store the email so we can delete if a GDPR request is made. We cannot do anything else with it.'
+      table.schema do |s|
+        s.string  "team_id",       mode: :required
+        s.string  "user_id",       mode: :required
+        s.string  "name",          mode: :required
+        s.string  "real_name",     mode: :required
+        s.string  "display_name",  mode: :required
+        s.boolean "is_owner",      mode: :required
+        s.boolean "is_admin",      mode: :required
+        s.string  "title"
+        s.string  "phone"
+        s.string  "skype"
+        s.string  "email"
+        s.string  "tz"
+        s.string  "tz_offset"
+        s.string  "status_text",   mode: :required
+        s.string  "status_emoji",  mode: :required
+        s.string  "image_72",      mode: :required
+        s.string  "image_192",     mode: :required
+        s.integer "updated",       mode: :required
+        s.boolean "deleted",       mode: :required
+      end
+    end
+    $logger.info "Bigquery initialization complete"
+  end
+
+  def get_auth_tokens()
+    @dataset.query("SELECT * FROM slack_teams").to_a rescue []
+  end
+
+  def insert_team(auth_token)
+    # We can take the 2.5s delete-before-insert here because it only happens when
+    # authorizing a new team integration.
+    @dataset.query("DELETE FROM slack_teams WHERE team_id = '#{auth_token[:team_id]}'")
+    @teams.insert(auth_token).success?
+  end
+
+  def insert_channel(record)
+    @channels.insert(record).success?
+  end
+
+  def insert_user(record)
+    @users.insert(record).success?
+  end
+
+  def insert_message(record)
+    @messages.insert(record).success?
+  end
+
+  def newest_message_timestamp
+    @dataset.query("SELECT ts FROM slack_messages ORDER BY ts DESC LIMIT 1").first[:ts] rescue 0
+  end
+
+  def record_user_creation(team_id, user_id)
+    @creation.insert(:team_id  => team_id,
+                     :user_id  => user_id,
+                     :creation => Time.now.strftime('%Y-%m-%d %H:%M:%S%z')).success?
+  end
+
+  # Ensure that all users have creation dates. If they don't have one yet, assume now
+  # This will catch any users who've been created during downtime if we restart the service
+  def reconcile_user_creations()
+    @dataset.query("INSERT INTO
+      slack_user_creation (team_id, user_id, creation)
+      (
+        SELECT team_id, user_id, CURRENT_TIMESTAMP()
+        FROM
+          slack_users
+        WHERE
+          user_id NOT IN
+          (
+            SELECT user_id from slack_user_creation
+          )
+      )")
+  end
+
+  def deduplicate_table(table)
+
+  end
+
+  def shell
+    require 'pry'
+    binding.pry
+  end
+end

data/lib/driftwood/slack.rb

@@ -0,0 +1,331 @@
+require 'slack-ruby-client'
+
+class Driftwood::Slack
+  def initialize(config)
+    @config   = config
+    @teams    = {}
+    @handlers = {}
+
+    $logger.info "Started Slack bot"
+
+#     # Example event handler. See https://api.slack.com/events
+#     register_handler('message') do |team_id, event_data|
+#       $logger.debug 'message handler'
+#       $logger.debug "Team ID: #{team_id}"
+#       $logger.debug JSON.pretty_generate(event_data)
+#
+#       next unless event_data['channel_type'] == 'im'
+#
+#       user = event_data['user']
+#       text = event_data['text']
+#
+#       case text
+#       when /^hello/i
+#         send_response(team_id, user, "Hi there, #{real_name(team_id, user)}!")
+#       when /^ping/i
+#         send_response(team_id, user, "Pong")
+#       when /^echo/i
+#         send_response(team_id, user, "Echo response: #{text.sub(/^echo /, '')}")
+#       end
+#     end
+  end
+
+  def authorize(auth)
+    team_id = nil
+    client  = Slack::Web::Client.new
+    # OAuth Step 3: Success or failure
+    # is this a cached authentication object?
+    if auth.is_a? Array
+      $logger.info "Using cached authentication for:"
+      auth.each do |row|
+        team_id         = row.delete(:team_id)
+        @teams[team_id] = row
+        $logger.info "  * #{row[:name] || team_id}"
+      end
+
+    # we have a token from a single oauth exchange
+    elsif auth.is_a? String
+      begin
+        response = client.oauth_access(
+          {
+            client_id:     @config[:client_id],
+            client_secret: @config[:client_secret],
+            redirect_uri:  @config[:redirect_uri],
+            code:          auth
+          }
+        )
+        team_id = response['team_id']
+        $logger.info "Authenticated new integration for team '#{team_id}'."
+      rescue Slack::Web::Api::Error => e
+        # Failure:
+        # D'oh! Let the user know that something went wrong and output the error message returned by the Slack client.
+        $logger.warn "Slack Authentication failed! Reason: #{e.message}"
+        raise "Slack Authentication failed! Reason: #{e.message}"
+      end
+
+      # Yay! Auth succeeded! Let's store the tokens
+      @teams[team_id] = {
+        :user_access_token => response['access_token'],
+        :bot_user_id       => response['bot']['bot_user_id'],
+        :bot_access_token  => response['bot']['bot_access_token']
+      }
+    else
+      raise "The authentication must either be a cached auth object or a token, not #{auth.class}"
+    end
+
+    # Create each Team's bot user and authorize the app to access the Team's Events.
+    @teams.each do |team_id, team|
+      begin
+        team[:client] = create_slack_client(team[:bot_access_token])
+        team[:user]   = create_slack_client(team[:user_access_token])
+        team[:name]   = team_name(team_id)
+        $logger.info "Authenticated tokens for team '#{team[:name]}'."
+        $logger.debug JSON.pretty_generate(@teams[team_id])
+      rescue Slack::Web::Api::Error => e
+        $logger.error "Cached authentication failed for team '#{team[:name] || team_id}'."
+      end
+    end
+
+    # return the hash structure w/o the client object for caching
+    team(team_id)
+  end
+
+  def register_handler(event, &block)
+    raise ArgumentError, "Handler (#{event}): no block passed" unless block_given?
+    raise ArgumentError, "Handler (#{event}): incorrect parameters" unless block.parameters.length == 2
+
+    @handlers[event] ||= Array.new
+    @handlers[event]  << block
+
+    $logger.info "Registered slack handler for #{event}"
+    $logger.debug 'Current handlers:'
+    $logger.debug @handlers.inspect
+  end
+
+  def create_slack_client(slack_api_secret)
+    Slack.configure do |config|
+      config.token = slack_api_secret
+      fail 'Missing API token' unless config.token
+    end
+    Slack::Web::Client.new
+  end
+
+  def event(request_data)
+    # Check the verification token provided with the request to make sure it matches the verification token in
+    # your app's setting to confirm that the request came from Slack.
+    unless @config[:verification_token] == request_data['token']
+      $logger.warn "Invalid Slack verification token received: #{request_data['token']}"
+      raise "Invalid Slack verification token received: #{request_data['token']}"
+    end
+
+    case request_data['type']
+    # When you enter your Events webhook URL into your app's Event Subscription settings, Slack verifies the
+    # URL's authenticity by sending a challenge token to your endpoint, expecting your app to echo it back.
+    # More info: https://api.slack.com/events/url_verification
+    when 'url_verification'
+      $logger.debug "Authorizing challenge: #{request_data['challenge']}"
+      return request_data['challenge']
+
+    when 'event_callback'
+      # Get the Team ID and Event data from the request object
+      team_id    = request_data['team_id']
+      event_data = request_data['event']
+      event_type = event_data['type']
+      user_id    = event_data['user']
+
+      # Don't process events from our bot user
+      unless user_id == @teams[team_id][:bot_user_id]
+        if @handlers.include? event_type
+          begin
+            @handlers[event_type].each do |handler|
+              handler.call(team_id, event_data)
+            end
+          rescue => e
+            $logger.error "Handler for #{event_type} failed: #{e.message}"
+            $logger.debug e.backtrace.join("\n")
+          end
+        else
+          $logger.info "Unhandled event:"
+          $logger.debug JSON.pretty_generate(request_data)
+        end
+      end
+
+    end
+    'ok'
+  end
+
+  # Send a response to an Event via the Web API.
+  def send_response(team_id, channel, text, unfurl_links = false, attachment = nil, ts = nil)
+    # `ts` is optional, depending on whether we're sending the initial
+    # welcome message or updating the existing welcome message tutorial items.
+    # We open a new DM with `chat.postMessage` and update an existing DM with
+    # `chat.update`.
+    if ts
+      @teams[team_id][:client].chat_update(
+        as_user: 'true',
+        channel: channel,
+        ts: ts,
+        text: text,
+        attachments: attachment,
+        unfurl_links: unfurl_links,
+      )
+    else
+      @teams[team_id][:client].chat_postMessage(
+        as_user: 'true',
+        channel: channel,
+        text: text,
+        attachments: attachment,
+        unfurl_links: unfurl_links,
+      )
+    end
+  end
+
+  def team(team_id)
+    teams.select {|team| team[:team_id] == team_id }.first
+  end
+
+  def teams()
+    # munge into an array of hashes, easier for external tools to use.
+    # ensure that the client object is removed to prevent serialization issues
+    # This weird copy-like thing is to prevent mutation of the original object
+    @teams.map do |team_id, team|
+      {:team_id => team_id}.merge(team.reject { |key,value| key == :client})
+    end
+  end
+
+  def channels(team_id)
+    preserve = ['team_id', 'channel_id', 'name', 'created', 'is_private', 'topic', 'purpose', 'num_members']
+    channels = @teams[team_id][:client].conversations_list(:types => 'public_channel, private_channel')
+    channels = channels['channels'].reject{|i| i['is_archived']}
+    channels.each do |row|
+      row['team_id']    = team_id
+      row['topic']      = row['topic']['value']
+      row['purpose']    = row['purpose']['value']
+      row['channel_id'] = row.delete('id')
+      row.select! {|field| preserve.include? field}
+    end
+  end
+
+  def users(team_id)
+    users  = []
+    cursor = nil
+    loop do
+      data   = @teams[team_id][:client].users_list(limit: 500, cursor: cursor)
+      cursor = data['response_metadata']['next_cursor']
+      users.concat data['members']
+
+      break if (cursor.nil? or cursor.empty?)
+    end
+
+    users.each do |row|
+      normalize_user(row)
+    end
+  end
+
+  def messages(team_id, starting_from = 0)
+    messages = []
+    channels = channels(team_id).map{|c| c['channel_id']}
+
+    channels.each do |channel_id|
+      oldest = starting_from
+      data   = { 'has_more' => true }
+
+      while data['has_more'] do
+        # note that we need the *user* scope here
+        data   = @teams[team_id][:user].channels_history(:channel => channel_id,
+                                                         :count   => 1000,
+                                                         :oldest  => oldest)
+
+        oldest = data['messages'].last['ts'] rescue oldest
+        messages.concat normalize_messages(team_id, channel_id, data['messages'])
+      end
+    end
+    messages
+  end
+
+  def all_channels
+    @teams.keys.inject([]) do |channels, team_id|
+      channels.concat(channels(team_id))
+    end
+  end
+
+  def all_users
+    @teams.keys.inject([]) do |users, team_id|
+      users.concat(users(team_id))
+    end
+  end
+
+  def all_messages(starting_from = 0)
+    @teams.keys.inject([]) do |messages, team_id|
+      messages.concat(messages(team_id, starting_from))
+    end
+  end
+
+  def user_info(team_id, user_id)
+    @teams[team_id][:client].users_info(user: user_id)['user']
+  end
+
+  def channel_info(team_id, channel_id)
+    @teams[team_id][:client].channels_info(channel: channel_id)['channel']
+  end
+
+  def team_info(team_id)
+    @teams[team_id][:client].team_info()['team']
+  end
+
+
+  def real_name(team_id, user_id)
+    user_info(team_id, user_id)['real_name'] rescue user_id
+  end
+
+  def channel_name(team_id, channel_id)
+    channel_info(team_id, channel_id)['name'] rescue channel_id
+  end
+
+  def team_name(team_id)
+    team_info(team_id)['name'] rescue team_id
+  end
+
+  def normalize_user(user)
+    preserve_fields = [ 'team_id', 'user_id', 'name', 'real_name','display_name',
+                        'updated', 'deleted', 'tz', 'tz_offset', 'is_owner', 'is_admin',
+                        'status_text', 'status_emoji', 'image_72', 'image_192',
+                      ]
+    # Why slack? Why?
+    preserve_fields.concat ['title', 'phone', 'skype', 'email'] unless user['deleted']
+
+    user['user_id']      = user.delete('id')
+    user['title']        = user['profile']['title']
+    user['phone']        = user['profile']['phone']
+    user['skype']        = user['profile']['skype']
+    user['email']        = user['profile']['email']
+    user['display_name'] = user['profile']['display_name']
+    user['status_text']  = user['profile']['status_text']
+    user['status_emoji'] = user['profile']['status_emoji']
+    user['image_72']     = user['profile']['image_72']
+    user['image_192']    = user['profile']['image_192']
+    user.select! {|field| preserve_fields.include? field}
+  end
+
+  def normalize_message(team_id, message)
+    preserve_fields = [ 'team_id', 'user_id', 'text', 'channel_id','ts' ]
+
+    message['team_id']    = team_id
+    message['user_id']    = message.delete('user')
+    message['channel_id'] = message.delete('channel')
+    message.select! {|field| preserve_fields.include? field}
+  end
+
+  def normalize_messages(team_id, channel_id, messages)
+    messages.map do |message|
+      message['channel'] = channel_id
+      normalize_message(team_id, message)
+    end
+  end
+
+
+  def shell
+    require 'pry'
+    binding.pry
+  end
+end

data/lib/driftwood.rb

@@ -0,0 +1,144 @@
+require 'json'
+require 'logger'
+require 'sinatra/base'
+
+class Driftwood < Sinatra::Base
+  require 'driftwood/slack'
+  require 'driftwood/bigquery'
+
+  set :logging, true
+  set :strict,  true
+  set :views,         File.dirname(__FILE__) + '/../views'
+  set :public_folder, File.dirname(__FILE__) + '/../public'
+
+  # this allows other tools to register handlers and interact with the db
+  attr_reader :slack, :bigquery
+
+  def initialize(app=nil)
+    super(app)
+    @bigquery = Driftwood::Bigquery.new(settings.gcloud)
+    @slack    = Driftwood::Slack.new(settings.slack)
+    @slack.authorize(cached_auth)
+
+    @slack.register_handler('message') do |team_id, event_data|
+      $logger.debug 'message handler'
+      $logger.debug "Team ID: #{team_id}"
+      $logger.debug "Channel Name: #{@slack.channel_name(team_id, event_data['channel'])}"
+      $logger.debug JSON.pretty_generate(event_data)
+
+      next unless event_data['channel_type'] == 'im'
+
+      user = event_data['user']
+      text = event_data['text']
+
+      case text
+      when /^hello/i
+        @slack.send_response(team_id, user, "Hi there, #{@slack.real_name(team_id, user)}!")
+      when /^ping/i
+        @slack.send_response(team_id, user, "Pong")
+      when /^echo/i
+        @slack.send_response(team_id, user, "Echo response: #{text.sub(/^echo /, '')}")
+      end
+
+    end
+
+    @slack.register_handler('team_join') do |team_id, event_data|
+      user = @slack.normalize_user(event_data['user'])
+      $logger.info "Registered new user: #{user['name']} (#{user['real_name']})"
+      @bigquery.insert_user(user)
+    end
+
+    @slack.register_handler('message') do |team_id, event_data|
+      next unless event_data['channel_type'] == 'channel'
+      $logger.debug "Logged message for #{team_id}: #{event_data['text']}"
+      @bigquery.insert_message(@slack.normalize_message(team_id, event_data))
+    end
+
+    # Do this in a thread so it doesn't block initialization
+    # These methods may take a long time to complete
+    Thread.new do
+      begin
+        $logger.info "Starting data synchronization..."
+        sync_channels
+        sync_users
+        sync_messages
+      rescue => e
+        $logger.error e.message
+        $logger.debug e.backtrace.join("\n")
+      end
+    end
+
+  end
+
+  get '/' do
+    if params[:code]
+      begin
+        save_auth_cache(@slack.authorize(params[:code]))
+        @message = "Rad, you've been authenticated!"
+        status 200
+
+      rescue => e
+        $logger.error e.message
+        $logger.debug e.backtrace.join("\n")
+
+        status 403
+        @error = e.message
+      end
+    end
+
+    erb :index
+  end
+
+  post '/events' do
+    begin
+      # Return HTTP status code 200 so Slack knows we've received the Event
+      status 200
+      body @slack.event(JSON.parse(request.body.read))
+    rescue => e
+      $logger.error e.message
+      $logger.debug e.backtrace.join("\n")
+      status 403
+      body "Error: #{e.message}"
+    end
+  end
+
+
+  not_found do
+    halt 404, "You shall not pass! (page not found)\n"
+  end
+
+  helpers do
+    def cached_auth
+      @bigquery.get_auth_tokens()
+    end
+
+    def save_auth_cache(auth)
+      @bigquery.insert_team(auth)
+    end
+
+    def sync_channels
+      @slack.all_channels.each do |channel|
+        @bigquery.insert_channel(channel)
+      end
+      $logger.info "All channels synced."
+    end
+
+    def sync_users
+      @slack.all_users.each do |user|
+        @bigquery.insert_user(user)
+      end
+      @bigquery.reconcile_user_creations
+      $logger.info "All users synced."
+    end
+
+    def sync_messages
+      starting_from = @bigquery.newest_message_timestamp
+      @slack.all_messages(starting_from).each do |message|
+        @bigquery.insert_message(message)
+      end
+      $logger.info "All messages synced."
+    end
+
+  end
+
+end

data/views/index.erb

@@ -0,0 +1,83 @@
+<html
+  <head>
+  <title>Driftwood</title>
+    <style>
+      body {
+        background-color: #f7fafd;
+      }
+      div.content {
+        width: 80%;
+        padding: 2em;
+        margin: 2em auto;
+        border: 2px ridge #a29999;
+        border-radius: 1em;
+        box-shadow: 5px 10px 8px #888888;
+        background-color: #efefef;
+        background-image: url(driftwood.jpg);
+        background-position: center;
+      }
+      h1, h2 {
+        text-align: center;
+        text-shadow: #ffe896 1px 0 10px;
+      }
+      p {
+        width: 80%;
+        margin: 1em auto;
+        text-shadow: #7f7d7e 1px 0 10px;
+      }
+      p.message {
+        width: 50%;
+        margin: 3em auto;
+        padding: 1em;
+        border-radius: 0.5em;
+        border: 2px solid #207c2b;
+        background-color: #f6f6f6;
+      }
+      p.error {
+        border: 2px solid red;
+      }
+      a#slack {
+        display: block;
+        margin: 0 auto;
+        text-align: center;
+      }
+    </style>
+	</head>
+	<body>
+  	<div class="content">
+    	<h1>Driftwood</h1>
+    	<h2>Extensible Slack Logbot</h2>
+    	<p>
+      	The Slack logging service isn't very complete, especially if you're on the free
+        tier. For example, messages expire far too soon on an active channel and users
+        don't include creation times. This tool provides those missing features, and soon
+        will also provide features for building a positive team culture.
+      </p>
+      <p>
+        Out of the box, Driftwood will log channels, users, and messages for Slack
+        teams. It also generates & tracks some metadata that the Slack API doesn't
+        provide, such as user account age. The database backend is currently limited
+        to Google BigQuery only, which makes it very easy to connect it to data
+        visualization dashboards, such as Looker.
+      </p>
+      <p>
+        Driftwood is extensible and allows you to easily hook into any Slack event
+        notifications. It's currently a relatively manual process to do so, but we'll
+        soon have a real plugin architecture and will then be able to list out each
+        integration installed below.
+      </p>
+      <hr />
+    <% if @message %>
+      <p class="message"><%= @message %></p>
+    <% end %>
+    <% if @error %>
+      <p class="message error"><%= @error %></p>
+    <% end %>
+      <p>
+        <a id="slack" href="https://slack.com/oauth/authorize?scope=bot%20channels:history&client_id=<%= settings.slack[:client_id] %>&redirect_uri=<%= settings.slack[:redirect_uri] %>">
+          <img alt="Add to Slack" height="40" width="139" src="https://platform.slack-edge.com/img/add_to_slack.png"/>
+        </a>
+        </p>
+  	</div>
+  </body>
+</html>

data/views/policy_updates.erb

@@ -0,0 +1,7 @@
+Hi <%= @user[:real_name] %>!
+
+We know you hate these as much as we do, but sometimes we need to notify you of certain changes. We'll keep it super brief.
+<% @updates.each do |row| %>
+We've updated our <%= row[:name] %>: <%= row[:summary] %>
+See more at <%= row[:url] %>
+<% end %>

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: driftwood
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ben Ford
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-07-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slack-ruby-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |2
+      The Slack logging service isn't very complete, especially if you're on the free
+      tier. For example, messages expire far too soon on an active channel and users
+      don't include creation times. This tool provides those missing features, and soon
+      will also provide features for building a positive team culture.
+email: ben.ford@puppet.com
+executables:
+- driftwood
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- bin/driftwood
+- lib/driftwood.rb
+- lib/driftwood/bigquery.rb
+- lib/driftwood/slack.rb
+- public/driftwood.jpg
+- views/index.erb
+- views/policy_updates.erb
+homepage: https://github.com/puppetlabs/driftwood
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.10
+signing_key: 
+specification_version: 4
+summary: Simple Sinatra based Slack logbot.
+test_files: []