embulk-input-slack_channel_messages 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 5d70c06c882e105c8b5729d24ad6e3d244d1f9f7048586b1da4039954cf63800
+  data.tar.gz: ba6594949b3ba80bfb9555b6ef09502d9dfa29e92ee81a9d6ce9c61c3021b1c1
+SHA512:
+  metadata.gz: 9d469dee6d30a5eb141ddebe98a97fc5c88283b57eaa51a0633c172e1a5dc64d3413b008db2884d8c0b364cb2fbd9c093ea3848dfc08b7350daa22f3d3f63e5a
+  data.tar.gz: b734b76cadf28c24f55aac1642fc3f7ea344e531f2695b38b3a744d9f54b7695688a1de6b5c1c865b3b86dd708c30c6892e0a49977700ffa0bf587104992204c

data/.gitignore

@@ -0,0 +1,7 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/.ruby-version
+/Gemfile.lock
+/spec/lib/embulk/input/slack/config.rb

data/.rspec

@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# Slack Channel Messages input plugin for Embulk
+
+input all messages from slack public channels
+
+## Overview
+
+**Plugin type**: input
+
+## Configuration
+
+| name        | type   | requirement | default | description              |
+| ----------- | ------ | ----------- | ------- | ------------------------ |
+| token       | string | 　 required |         | slack api token          |
+| channel_ids | array  | required    |         | slack public channel ids |
+
+using slack apis below
+
+- auth.test
+- users.list
+- conversations.history
+- conversations.info
+
+**We must add an OAuth scopes for using these apis.**
+
+## Example
+
+```yaml
+in:
+  type: slack_channel_messages
+  token: <slack api token>
+  channel_ids:
+    - <public channel id 1>
+    - <public channel id 2>
+```
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-input-slack_channel_messages.gemspec

@@ -0,0 +1,21 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-slack_channel_messages"
+  spec.version       = "0.1.0"
+  spec.authors       = ["Sasaki1994"]
+  spec.summary       = "Slack Channel Messages input plugin for Embulk"
+  spec.description   = "Loads slack channel messages using slack API."
+  spec.email         = ["b.92421933411@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/Sasaki1994/embulk-input-slack_channel_messages"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  # spec.add_development_dependency 'embulk', ['>= 0.9.23']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'rspec'
+end

data/lib/embulk/input/slack/client.rb

@@ -0,0 +1,118 @@
+require 'net/http'
+require 'uri'
+require 'json'
+
+module Slack
+    class Client
+      attr_reader :token
+  
+      def initialize(token)
+        @token = token
+      end
+  
+      def ch_messages(channel_id)
+        """
+        Get all channel messages by channel_id using conversations.history api.
+        This API can only retrieve up to 1000 at a time, so we use 'next_cusor' to retrieve them recursively.
+
+        conversations.history api
+        https://api.slack.com/methods/conversations.history
+
+        params: slack public channel id (string)
+        return: all channel messages with datetime and user id (Array[Array])
+        """
+        messages = []
+        cursor = nil
+        begin
+            res = JSON.parse(get('conversations.history', {channel: channel_id, limit: 1000, cursor: cursor}))
+            
+            messages += res['messages'].select{ |message| message['type'] == 'message' }.map do |message|
+                {
+                    'timestamp' => message['ts'],
+                    'user_id' => message['user'],
+                    'text' => message['text']
+                }
+            end
+            
+            cursor = res['response_metadata']&.[]('next_cursor') || ''
+        end until cursor == ''
+        messages
+      end
+  
+      def members
+        """
+        Get all members using users.list api.
+        This API is limited at a time, so we use 'next_cusor' to retrieve them recursively.
+
+        users.list api
+        https://api.slack.com/methods/users.list
+
+        return: all user id(key) and name(value) store (Hash)
+        """
+        members = {}
+        cursor = nil
+        begin
+            res = JSON.parse(get('users.list', {limit: 1000, cursor: cursor}))
+            raise "Error '#{res['error']}'" unless res['ok']
+            
+            res['members'].each do |member|
+                members[member['id']] = member['profile']['display_name']
+            end
+            
+            cursor = res['response_metadata']&.[]('next_cursor') || ''
+        end until cursor == ''
+        members
+      end
+  
+      def channel_by_id!(ch_id)
+        """
+        Get channel information by channel_id using conversations.info api.
+
+        conversations.info api
+        https://api.slack.com/methods/conversations.info
+
+        params: slack public channel id (string)
+        return: channel object (hash) 
+        """
+        res = JSON.parse(get('conversations.info', {channel: ch_id}))
+        raise "Error #{res['error']} channel id = #{ch_id}" unless res['ok']
+        res['channel']
+      end
+
+      def channel_by_id(ch_id)
+        res = JSON.parse(get('conversations.info', {channel: ch_id}))
+        return nil unless res['ok']
+        res['channel']
+      end
+
+      def auth_check
+        """
+        Do auth check test using auth.test api.
+
+        auth.test api
+        https://api.slack.com/methods/auth.test
+
+        return: is successed to api request with @token auth (bool)
+        """
+        res = JSON.parse(post('auth.test'))
+        raise 'Slack API Server Error' unless res.has_key?("ok")
+        res["ok"]
+      end
+
+      private
+      def slack_uri(api, params=nil)
+        uri = URI.parse("https://slack.com/api/#{api}")
+        uri.query = URI.encode_www_form(params) if params
+        uri
+      end
+
+      def get(api, options = {})
+        Net::HTTP.get(slack_uri(api, options.merge(token: token)))
+      end
+
+      def post(api, options = {})
+        Net::HTTP.post_form(slack_uri(api), options.merge(token: token)).body
+      end
+    end
+end
+  

data/lib/embulk/input/slack/service.rb

@@ -0,0 +1,58 @@
+require_relative 'client'
+module Slack
+    class Service
+        attr_reader :token
+        
+        def initialize(token)
+            @token = token
+            @client = Client.new(@token)
+        end
+
+        def compose_record(channel_id, &block)
+            """
+            Compose embulk insert record.
+            
+            params:
+                slack public channel id (string)
+                embulk insert method (block)
+            """
+            members = get_members
+            channel = get_channel(channel_id)
+            messages = get_messages(channel_id)
+
+            messages.each do |message|
+                record = {
+                    'id' => "#{channel_id}-#{message['timestamp'].to_i}-#{message['user_id']}",
+                    'post_at' => Time.at(message['timestamp'].to_i),
+                    'slack_user_id' => message['user_id'],
+                    'slack_username' => members[message['user_id']],
+                    'slack_channel_name' => channel['name'],
+                    'message_text' => message['text']
+                }
+                block.call(record)
+            end
+        end
+
+        def get_members
+            @client.members
+        end
+
+        def get_channel(channel_id)
+            @client.channel_by_id!(channel_id)
+        end
+
+        def get_messages(channel_id)
+            @client.ch_messages(channel_id)
+        end
+
+        def check_token
+            raise 'Error token is invalid' unless @client.auth_check
+        end
+
+        def check_channels(channel_ids)
+            channel_ids.each do |ch_id|
+                @client.channel_by_id!(ch_id)
+            end
+        end
+    end
+end

data/lib/embulk/input/slack_channel_messages.rb

@@ -0,0 +1,70 @@
+require_relative 'slack/service'
+
+module Embulk
+  module Input
+
+    class SlackChannelMessages < InputPlugin
+      Plugin.register_input("slack_channel_messages", self)
+      
+      @@columns = [
+        ["id", :string],
+        ["post_at", :timestamp],
+        ["slack_user_id", :string],
+        ["slack_username", :string],
+        ["slack_channel_name", :string],
+        ["message_text", :string],
+      ]
+
+      def self.transaction(config, &control)
+        # configuration code:
+        task = {
+          "token" => config.param("token", :string),
+          "channel_ids" => config.param("channel_ids", :array),
+        }
+
+        columns = @@columns.map.with_index{ |column, index| Column.new(index, *column) }
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        task_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      # TODO
+      # def self.guess(config)
+      #   sample_records = [
+      #     {"example"=>"a", "column"=>1, "value"=>0.1},
+      #     {"example"=>"a", "column"=>2, "value"=>0.2},
+      #   ]
+      #   columns = Guess::SchemaGuess.from_hash_records(sample_records)
+      #   return {"columns" => columns}
+      # end
+
+      def init
+        @token = task["token"]
+        @channel_ids = task["channel_ids"]
+        @service = Slack::Service.new(@token)
+      end
+
+      def run
+        @service.check_token
+        @service.check_channels(@channel_ids)
+        @channel_ids.each do |channel_id|
+          @service.compose_record(channel_id) do |record|
+            # ex) page_builder.add([<col1>, <col2>, <col3> ...])
+            page_builder.add(@@columns.map{|col| record[col[0]]})
+          end
+        end
+        page_builder.finish
+
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

data/spec/lib/embulk/input/slack/client_spec.rb

@@ -0,0 +1,77 @@
+require 'embulk/input/slack/client'
+require_relative 'config'
+
+RSpec.describe do
+
+  let!(:client) {Slack::Client.new(config[:token])}
+
+  describe '#ch_messages' do
+    let(:messages) {client.ch_messages(config[:channel])}
+
+    it 'is hash of array' do
+      expect(messages).to be_an_instance_of(Array)
+      expect(messages[0]).to be_an_instance_of(Hash)
+    end
+
+    it 'has expected keys' do
+      expect(messages[0].keys).to include("timestamp", "user_id", "text")
+    end
+  end
+
+  describe '#members' do
+    let(:members) {client.members}
+
+    it 'is hash' do
+      expect(members).to be_an_instance_of(Hash)
+    end
+  end
+
+  describe '#channel_by_id!' do
+    let(:channel) {client.channel_by_id!(config[:channel])}
+
+    it 'is hash' do
+      expect(channel).to be_an_instance_of(Hash)
+    end
+
+    it 'has "name" key' do
+      expect(channel.keys).to include("name")
+    end
+
+    context 'not found channel id' do
+      it 'occurs error' do
+        expect{client.channel_by_id!('no_id')}.to raise_error('Error channel_not_found channel id = no_id')
+      end
+    end
+  end
+
+  describe '#channel_by_id' do
+    let(:channel) {client.channel_by_id(config[:channel])}
+
+    it 'is hash' do
+      expect(channel).to be_an_instance_of(Hash)
+    end
+
+    it 'has "name" key' do
+      expect(channel.keys).to include("name")
+    end
+
+    context 'not found channel id' do
+      it 'returns nil' do
+        expect(client.channel_by_id('no_id')).to eq nil
+      end
+    end
+  end
+
+  describe '#auth_check' do
+    it 'returns true' do
+      expect(client.auth_check).to eq true
+    end
+
+    context 'token is invalid' do
+      it 'returns false' do
+        client = Slack::Client.new("invalid token")
+        expect(client.auth_check).to eq false
+      end
+    end
+  end
+end

data/spec/lib/embulk/input/slack/service_spec.rb

@@ -0,0 +1,30 @@
+require 'embulk/input/slack/service'
+
+RSpec.describe do
+
+  describe '#compose_record' do
+    it 'A record is correct hash' do
+      service = Slack::Service.new("token")
+      allow(service).to receive(:get_members).and_return({"A00AAA00" => "sample_user"})
+      allow(service).to receive(:get_channel).and_return({"name" => "sample_channel"})
+      allow(service).to receive(:get_messages).and_return([{
+        'timestamp' => 111111.1111,
+        'user_id' => "A00AAA00",
+        'text' => "sample message"
+      }]) 
+
+      ex_record = {
+          'id' => "channel_id-111111-A00AAA00",
+          'post_at' => Time.at(111111),
+          'slack_user_id' => "A00AAA00",
+          'slack_username' => "sample_user",
+          'slack_channel_name' => "sample_channel",
+          'message_text' => "sample message"
+      }
+      
+      service.compose_record("channel_id") do |record|
+        expect(record).to eq ex_record
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,100 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause
+# this file to always be loaded, without a need to explicitly require it in any
+# files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, consider making
+# a separate helper file that requires the additional dependencies and performs
+# the additional setup, and require it from the spec files that actually need
+# it.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+  # This option will default to `:apply_to_host_groups` in RSpec 4 (and will
+  # have no way to turn it off -- the option exists only for backwards
+  # compatibility in RSpec 3). It causes shared context metadata to be
+  # inherited by the metadata hash of host groups and examples, rather than
+  # triggering implicit auto-inclusion in groups with matching metadata.
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # This allows you to limit a spec run to individual examples or groups
+  # you care about by tagging them with `:focus` metadata. When nothing
+  # is tagged with `:focus`, all examples get run. RSpec also provides
+  # aliases for `it`, `describe`, and `context` that include `:focus`
+  # metadata: `fit`, `fdescribe` and `fcontext`, respectively.
+  config.filter_run_when_matching :focus
+
+  # Allows RSpec to persist some state between runs in order to support
+  # the `--only-failures` and `--next-failure` CLI options. We recommend
+  # you configure your source control system to ignore this file.
+  config.example_status_persistence_file_path = "spec/examples.txt"
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/
+  #   - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = "doc"
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-slack_channel_messages
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sasaki1994
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Loads slack channel messages using slack API.
+email:
+- b.92421933411@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-slack_channel_messages.gemspec
+- lib/embulk/input/slack/client.rb
+- lib/embulk/input/slack/service.rb
+- lib/embulk/input/slack_channel_messages.rb
+- spec/lib/embulk/input/slack/client_spec.rb
+- spec/lib/embulk/input/slack/service_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/Sasaki1994/embulk-input-slack_channel_messages
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Slack Channel Messages input plugin for Embulk
+test_files:
+- spec/lib/embulk/input/slack/client_spec.rb
+- spec/lib/embulk/input/slack/service_spec.rb
+- spec/spec_helper.rb