logstash-input-twitter 4.0.3 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90a556dabd75cd5e9260c327a7bdffa6bfe5e7fe4f80457901505953d7dc31f2
-  data.tar.gz: c83225608136218ea6b950f158fcd1983b3455c533f37a4f01d768b1e6dc03e2
+  metadata.gz: 35aa088d3d7f0ebcecf9ea2e00ea9009dacf682b99ca57f544ca25201cf70744
+  data.tar.gz: 5e740cbcabb04ba97a218324d8d5c56aa20f992382347e3aaae29bf4ff6260b8
 SHA512:
-  metadata.gz: cced602c79e40db401fb8a2547fc2a4756a9680b62ce31852f953bcc7a17942076378e04e403505403152f8d751b9a61f90d643ff406aa3cbc90e23438291fbe
-  data.tar.gz: 7e2cef7a66efcd33566dab11d6e293b07a6a3458d22c66372b28f63dcec676dcba2fff6e63c5aec7f41836109afda8d4233f1a4c7e76625c4bbbd8fff71c8900
+  metadata.gz: 61c1343445e3dca087247221044d41feb05c660d17682cb47bf0499f31f19a5206f636128f67c4d1c34295db29bd45c305344d63cef09ed10b7a5cf9dbc9514d
+  data.tar.gz: 2dbb9da7f8097ec766bf2ad0a2ae3745e7e811b31f5e8dded9b8fac413edffd014eaadc247895373d1260ee0a690a6b961fe17a50ec3a01d882d02c859643f97

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+ - Feat: optional target + ecs_compatibility [#72](https://github.com/logstash-plugins/logstash-input-twitter/pull/72)
+
 ## 4.0.3
  - Fix: broken proxy configuration [#69](https://github.com/logstash-plugins/logstash-input-twitter/pull/69)
 

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Plugin
 
-[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-input-twitter.svg)](https://travis-ci.org/logstash-plugins/logstash-input-twitter)
+[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-twitter.svg)](https://travis-ci.com/logstash-plugins/logstash-input-twitter)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 

data/docs/index.asciidoc

@@ -23,6 +23,46 @@ include::{include_path}/plugin_header.asciidoc[]
 
 Ingest events from the Twitter Streaming API.
 
+Example:
+[source,ruby]
+    input {
+      twitter {
+        consumer_key => '...'
+        consumer_secret => '...'
+        oauth_token => '...'
+        oauth_token_secret => '...'
+        keywords => [ 'logstash' ]
+      }
+    }
+
+Sample event fields generated:
+
+[source,ruby]
+-----
+    {
+        "@timestamp" => 2019-09-23T16:41:53.000Z,
+        "message" => "I forgot how fun it is to write @logstash configs !!! Thank you @jordansissel and @elastic !!!"
+        "user" => "missnebun",
+        "in-reply-to" => nil,
+        "retweeted" => false,
+        "source" => "http://twitter.com/missnebun/status/1176174859833004037",
+        "user_mentions" => [
+            { "screen_name"=>"logstash", "name"=>"logstash", "id"=>217155915 },
+            { "screen_name"=>"jordansissel", "name"=>"@jordansissel", "id"=>15782607 },
+            { "screen_name"=>"elastic", "name"=>"Elastic", "id"=>84512601 }],
+        "symbols" => [],
+        "hashtags" => [],
+        "client" => "<a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>"
+    }
+-----
+
+[id="plugins-{type}s-{plugin}-ecs"]
+==== Compatibility with the Elastic Common Schema (ECS)
+
+Twitter streams are very specific and do not map easily to the {ecs-ref}[Elastic Common Schema (ECS)].
+We recommend setting a <<plugins-{type}s-{plugin}-target>> when <<plugins-{type}s-{plugin}-ecs_compatibility,ECS compatibility mode>> is enabled.
+The plugin issues a warning in the log when a `target` isn't set.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Twitter Input Configuration Options
 
@@ -33,6 +73,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-consumer_key>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-consumer_secret>> |<<password,password>>|Yes
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-follows>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-full_tweet>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ignore_retweets>> |<<boolean,boolean>>|No
@@ -46,6 +87,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-rate_limit_reset_in>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-use_proxy>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-use_samples>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
 
 Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
@@ -78,6 +120,19 @@ If you don't have one of these, you can create one by
 registering a new application with Twitter:
 <https://dev.twitter.com/apps/new>
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+  * Value type is <<string,string>>
+  * Supported values are:
+    ** `disabled`: does not use ECS-compatible field names (fields might be set at the root of the event)
+    ** `v1`, `v8`: avoids field names that might conflict with Elastic Common Schema (for example, Twitter specific properties)
+  * Default value depends on which version of Logstash is running:
+    ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+    ** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+
 [id="plugins-{type}s-{plugin}-follows"]
 ===== `follows` 
 
@@ -221,6 +276,17 @@ by the default access level are the same, so if two different clients connect
 to this endpoint, they will see the same tweets. If set to true, the keywords, 
 follows, locations, and languages options will be ignored. Default => false
 
+[id="plugins-{type}s-{plugin}-target"]
+===== `target`
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Without a `target`, events are created from tweets at the root level.
+When the `target` is set to a field reference, the tweet data is placed in the target field instead.
+
+This option can be useful to avoid populating unknown fields when a downstream schema such as ECS is enforced.
+
 
 
 [id="plugins-{type}s-{plugin}-common-options"]

data/lib/logstash/inputs/twitter.rb

@@ -7,10 +7,20 @@ require "logstash/json"
 require "stud/interval"
 require "twitter"
 require "logstash/inputs/twitter/patches"
+require 'logstash/plugin_mixins/ecs_compatibility_support'
+require 'logstash/plugin_mixins/ecs_compatibility_support/target_check'
+require 'logstash/plugin_mixins/event_support/event_factory_adapter'
+require 'logstash/plugin_mixins/validator_support/field_reference_validation_adapter'
 
 # Ingest events from the Twitter Streaming API.
 class LogStash::Inputs::Twitter < LogStash::Inputs::Base
 
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
+  include LogStash::PluginMixins::ECSCompatibilitySupport::TargetCheck
+  include LogStash::PluginMixins::EventSupport::EventFactoryAdapter
+
+  extend LogStash::PluginMixins::ValidatorSupport::FieldReferenceValidationAdapter
+
   attr_reader :filter_options, :event_generation_error_count
 
   config_name "twitter"
@@ -104,6 +114,11 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
   # is nil. If this occurs then we use the integer specified here. The default is 5 minutes.
   config :rate_limit_reset_in, :validate => :number, :default => 300
 
+  # Defines a target field for placing fields.
+  # If this setting is omitted, data gets stored at the root (top level) of the event.
+  # The target is only relevant while decoding data into a new event.
+  config :target, :validate => :field_reference
+
   def register
     if !@use_samples && ( @keywords.nil? && @follows.nil? && @locations.nil? )
       raise LogStash::ConfigurationError.new("At least one parameter (follows, locations or keywords) must be specified.")
@@ -189,35 +204,34 @@ class LogStash::Inputs::Twitter < LogStash::Inputs::Base
 
   def from_tweet(tweet)
     @logger.debug? && @logger.debug("Got tweet", :user => tweet.user.screen_name, :text => tweet.text)
+
     if @full_tweet
-      event = LogStash::Event.new(LogStash::Util.stringify_symbols(tweet.to_hash))
-      event.timestamp = LogStash::Timestamp.new(tweet.created_at)
+      attributes = LogStash::Util.stringify_symbols(tweet.to_hash)
     else
-
       attributes = {
-        LogStash::Event::TIMESTAMP => LogStash::Timestamp.new(tweet.created_at),
         "message" => tweet.full_text,
         "user" => tweet.user.screen_name,
         "client" => tweet.source,
         "retweeted" => tweet.retweeted?,
-        "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}"
+        "source" => "http://twitter.com/#{tweet.user.screen_name}/status/#{tweet.id}",
+        "hashtags" => tweet.hashtags.map(&:attrs),
+        "symbols" => tweet.symbols.map(&:attrs),
+        "user_mentions" => tweet.user_mentions.map(&:attrs),
       }
 
-      attributes["hashtags"] = tweet.hashtags.map{|ht| ht.attrs}
-      attributes["symbols"]  = tweet.symbols.map{|sym| sym.attrs}
-      attributes["user_mentions"]  = tweet.user_mentions.map{|um| um.attrs}
-      event = LogStash::Event.new(attributes)
       if tweet.reply? && !tweet.in_reply_to_status_id.nil?
-        event.set("in-reply-to", tweet.in_reply_to_status_id)
+        attributes["in-reply-to"] = tweet.in_reply_to_status_id
       end
       unless tweet.urls.empty?
-        event.set("urls", tweet.urls.map(&:expanded_url).map(&:to_s))
+        attributes["urls"] = tweet.urls.map(&:expanded_url).map(&:to_s)
       end
     end
 
     # Work around bugs in JrJackson. The standard serializer won't work till we upgrade
     # event.set("in-reply-to", nil) if event.get("in-reply-to").is_a?(Twitter::NullObject)
 
+    event = targeted_event_factory.new_event(attributes)
+    event.timestamp = LogStash::Timestamp.new(tweet.created_at)
     event
   end
 

data/logstash-input-twitter.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-twitter'
-  s.version         = '4.0.3'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads events from the Twitter Streaming API"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -24,6 +24,9 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.3'
+  s.add_runtime_dependency "logstash-mixin-event_support", '~> 1.0'
+  s.add_runtime_dependency 'logstash-mixin-validator_support', '~> 1.0'
   s.add_runtime_dependency 'twitter', '6.2.0'
   s.add_runtime_dependency 'http-form_data', '~> 2'
   s.add_runtime_dependency 'public_suffix', '~> 3'

data/spec/inputs/twitter_spec.rb

@@ -97,6 +97,58 @@ describe LogStash::Inputs::Twitter do
 
   end
 
+  let(:tweet) do
+    Twitter::Tweet.new(id: 1234567890123456789,
+                       id_str: '1234567890123456789',
+                       created_at: 'Thu Nov 11 10:00:00 +0000 2021',
+                       text: 'Hello World! 🥰',
+                       source: "<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>",
+                       truncated: false,
+                       user: {
+                           id: 1182909005012345600,
+                           name: "Foo Bar",
+                           screen_name: "foo001",
+                           location: nil,
+                       })
+  end
+
+  describe "from tweet" do
+
+    before(:each) do
+      plugin.register
+      @event = plugin.send :from_tweet, tweet
+    end
+
+    it "generated an event" do
+      expect(@event.get('message')).to eql "Hello World! 🥰"
+      expect(@event.get('user')).to eql "foo001"
+      expect(@event.timestamp.to_s).to match /2021-11-11T10:00:00(.000)?Z/
+    end
+
+    context 'with full_tweet' do
+      let(:config) { super().merge 'full_tweet' => true }
+
+      it "generated an event" do
+        expect(@event.get('text')).to eql "Hello World! 🥰"
+        expect(@event.get('user')['screen_name']).to eql "foo001"
+        expect(@event.timestamp.to_s).to match /2021-11-11T10:00:00(.000)?Z/
+      end
+    end
+
+    context 'with target' do
+      let(:config) { super().merge 'target' => '[twitter]' }
+
+      it "generated an event" do
+        expect(@event.include?('message')).to be false
+        expect(@event.include?('user')).to be false
+
+        expect(@event.get('[twitter][message]')).to eql "Hello World! 🥰"
+        expect(@event.get('[twitter][user]')).to eql "foo001"
+        expect(@event.timestamp.to_s).to match /2021-11-11T10:00:00(.000)?Z/
+      end
+    end
+  end
+
   describe "stream filter" do
 
     describe "options parsing" do
@@ -161,8 +213,6 @@ describe LogStash::Inputs::Twitter do
 
       context "when not filtering retweets" do
 
-        let(:tweet) { Twitter::Tweet.new(id: 1) }
-
         let(:config) do
           {
             'consumer_key' => 'foo',
@@ -187,8 +237,6 @@ describe LogStash::Inputs::Twitter do
 
       context "when filtering retweets" do
 
-        let(:tweet) { Twitter::Tweet.new(id: 1) }
-
         let(:config) do
           {
             'consumer_key' => 'foo',
@@ -219,7 +267,7 @@ describe LogStash::Inputs::Twitter do
       end
 
       let(:config) do
-        super.merge( 'use_proxy' => true, 'proxy_address' => '127.0.0.1', 'proxy_port' => 12345 )
+        super().merge( 'use_proxy' => true, 'proxy_address' => '127.0.0.1', 'proxy_port' => 12345 )
       end
 
       let(:proxy_socket) { double('proxy-socket') }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-twitter
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-06 00:00:00.000000000 Z
+date: 2021-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,48 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  name: logstash-mixin-ecs_compatibility_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-event_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: logstash-mixin-validator_support
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -179,8 +221,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Reads events from the Twitter Streaming API