log-export-container 1.0.53

data/fluentd/etc/output-splunk-hec.conf

@@ -0,0 +1,12 @@
+<store>
+  @type               splunk_hec
+  hec_host            "#{ENV['SPLUNK_HEC_HOST']}"
+  hec_port            "#{ENV['SPLUNK_HEC_PORT']}"
+  hec_token           "#{ENV['SPLUNK_HEC_TOKEN']}"
+
+  # ssl params
+  insecure_ssl        true
+
+  # for more config options 
+  # see https://github.com/splunk/fluent-plugin-splunk-hec
+</store>

data/fluentd/etc/output-stdout.conf

@@ -0,0 +1,3 @@
+<store>
+  @type               stdout
+</store>

data/fluentd/etc/output-sumologic.conf

@@ -0,0 +1,10 @@
+<store>
+  @type               sumologic
+  endpoint            "#{ENV['SUMOLOGIC_ENDPOINT']}"
+  log_format          json
+  source_name         sdm
+  source_category     "#{ENV['SUMOLOGIC_SOURCE_CATEGORY']}"
+
+  # for more config options 
+  # see https://github.com/SumoLogic/fluentd-output-sumologic
+</store>

data/fluentd/etc/output-template.conf

@@ -0,0 +1,4 @@
+<match **>
+  @type                 copy
+  $stores
+</match>

data/fluentd/etc/process.conf

@@ -0,0 +1,19 @@
+# Config file for processing log traces
+
+# Sanitizer Rules
+# see https://github.com/fluent/fluent-plugin-sanitizer
+
+# <filter **>
+#   @type                     sanitizer
+#   hash_salt                 sdmsalt
+#   <rule> 
+#     keys                    query
+#     pattern_regex           /SET PASSWORD .*/
+#     pattern_regex_prefix    "CHANGE_PASSWORD"
+#   </rule>
+#   <rule> 
+#     keys                    query
+#     pattern_regex           /ALTER USER .+ IDENTIFIED BY .+/
+#     pattern_regex_prefix    "CHANGE_PASSWORD"
+#   </rule>
+# </filter>

data/fluentd/plugins/filter_sdm_decode_chunk_events.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_json'
+
+module Fluent::Plugin
+  class SDMDecodeChunkEventsFilter < Filter
+    Fluent::Plugin.register_filter('sdm_decode_chunk_events', self)
+
+    def filter(tag, time, record)
+      decode_chunk_log(record)
+    end
+
+    private
+
+    def decode_chunk_log(record)
+      decoded_events = []
+
+      full_cmd_entry = ''
+      total_elapsed_millis = 0
+      start_time_regular = zulu_date_to_regular(record['timestamp'])
+      begin
+        record['events'].each do |event|
+          duration, command = extract_cmd_entry_info(event)
+          one_line_cmd_entry = command.gsub("\r", '')
+          total_elapsed_millis += duration
+          full_cmd_entry = "#{full_cmd_entry}#{one_line_cmd_entry}"
+
+          next unless end_of_line(one_line_cmd_entry)
+
+          end_time_regular = add_millis(start_time_regular, total_elapsed_millis)
+
+          item = {
+            'data' => full_cmd_entry.split("\n"),
+            'startTimestamp' => start_time_regular,
+            'endTimestamp' => end_time_regular
+          }
+
+          decoded_events << item
+
+          full_cmd_entry = ''
+          total_elapsed_millis = 0
+          start_time_regular = end_time_regular
+        end
+
+        record['decodedEvents'] = decoded_events unless record['decodedEvents']
+      rescue StandardError => _e
+        puts "An error ocurred: #{_e.message}"
+      end
+
+      record
+    end
+
+    def end_of_line(line)
+      line.include? "\n"
+    end
+
+    def zulu_date_to_regular(input_date)
+      input_date.gsub(/[0-9]{,3} \+[0-9]{,4} UTC$/, '')
+    end
+
+    def extract_cmd_entry_info(line)
+      [line['duration'], Base64.decode64(line['data']).force_encoding('utf-8')]
+    end
+
+    def add_millis(input_date, input_millis)
+      new_date = Time.parse input_date
+      (new_date.to_time + input_millis / 1000.0).iso8601(3).to_s
+    end
+  end
+end

data/fluentd/plugins/parser_sdm_json.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'fluent/plugin/parser'
+require 'fluent/plugin/parser_json'
+
+# Remove characters before JSON from SDM log lines, format:
+# 2021-06-22T17:15:19Z ip-172-31-3-25 strongDM[734548]: {\"type\":\"complete\",\"timestamp\":\"2021-06-22T17:15:19.758785454Z\",\"uuid\":\"01uJSIaxJKEf6y85VRAoypiPJUGJ\",\"duration\":0,\"records\":1}
+module Fluent::Plugin
+  class SDMJsonParser < Parser
+    Fluent::Plugin.register_parser('sdm_json', self)
+
+    def configure(conf)
+      super
+      @parser = Fluent::Plugin::JSONParser.new
+      @parser.configure(Fluent::Config::Element.new('ROOT', '', {}, []))
+    end
+
+    def parse(text)
+      idx = text.index('{')
+      if idx
+        text = text[idx..-1]
+      end
+      @parser.parse(text) do |_, r|
+        text = r
+      end
+      yield Fluent::EventTime.now, text
+    end
+  end
+end

data/fluentd/scripts/dump_sdm_entities.rb

@@ -0,0 +1,117 @@
+require 'json'
+require 'date'
+require 'open3'
+require 'socket'
+
+SIGTERM = 15
+
+AUDIT_ENTITY_TYPES = {
+  "activities" => "activity",
+  "resources" => "resource",
+  "users" => "user",
+  "roles" => "role",
+}
+
+def get_audit_rows(entity_name)
+  if entity_name == "activities"
+    return get_audit_activities_rows
+  end
+  output = `sdm audit #{entity_name} -j`
+  output.split("\n")
+end
+
+def get_audit_activities_rows
+  interval_time = extract_activities_interval
+  if interval_time == nil
+    stream_activities
+    return
+  end
+  datetime_from = DateTime.now - (interval_time + 1.0)/(24*60)
+  datetime_to = DateTime.now - 1.0/(24*60)
+  output = `sdm audit activities -j -e --from "#{datetime_from.to_s}" --to "#{datetime_to.to_s}"`
+  output.split("\n")
+end
+
+def extract_activities_interval
+  extract_entities = ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT']
+  if ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL'] != nil
+    interval = ENV['LOG_EXPORT_CONTAINER_EXTRACT_AUDIT_ACTIVITIES_INTERVAL'].to_i
+  elsif extract_entities&.match /activities\/stream/
+    return nil
+  else
+    interval_match = extract_entities&.match /activities\/+(\d+)/
+    interval = interval_match ? interval_match[1].to_i : 15
+  end
+  interval
+end
+
+def stream_activities
+  stdout_and_stderr, thread = open_activities_stream
+  process_activity_stream(stdout_and_stderr)
+  Process.kill(SIGTERM, thread.pid)
+end
+
+def open_activities_stream
+  must_stream_json = ENV['LOG_EXPORT_CONTAINER_INPUT'].include?("json")
+  command = "sdm audit activities -e -f"
+  if must_stream_json
+    command += " -j"
+  end
+  _, stdout_and_stderr, thread = Open3.popen2e(command)
+  [stdout_and_stderr, thread]
+end
+
+def send_socket_message(message)
+  client = TCPSocket.open('localhost', 5140)
+  client.puts "<5>#{message}"
+  client.close
+end
+
+def parse_entity(entity, entity_name)
+  parsed_entity = JSON.parse(entity)
+  parsed_entity['type'] = AUDIT_ENTITY_TYPES[entity_name]
+  parsed_entity
+end
+
+def process_activity_stream(stdout)
+  must_stream_json = ENV['LOG_EXPORT_CONTAINER_INPUT'].include?("json")
+  stdout.each do |line|
+    if must_stream_json
+      message = JSON.generate(parse_entity(line, 'activities'))
+    else
+      message = line.gsub("\n", "") + ",activity"
+    end
+    send_socket_message(message)
+  end
+end
+
+def parse_rows(rows, entity_name)
+  parsed_rows = []
+  rows.each do |row|
+    parsed_rows << parse_entity(row, AUDIT_ENTITY_TYPES[entity_name])
+  end
+  parsed_rows
+end
+
+def print_rows(rows)
+  rows.each { |row| puts "#{JSON.generate(row)}" }
+end
+
+def dump_entities(entity_name)
+  unless AUDIT_ENTITY_TYPES.keys.include?(entity_name.to_s)
+    return
+  end
+  begin
+    rows = get_audit_rows(entity_name)
+    unless rows
+      return
+    end
+    parsed_rows = parse_rows(rows, entity_name)
+    print_rows(parsed_rows)
+  rescue StandardError => _e
+    error = {"error" => "An error ocurred while extracting the audit #{entity_name.to_s} data: #{_e}", "type" => "unclass"}
+    send_socket_message(JSON.generate(error))
+  end
+end
+
+dump_entities(ARGV[0])

data/start.rb

@@ -0,0 +1,34 @@
+::USING_WINDOWS = !!((RUBY_PLATFORM =~ /(win|w)(32|64)$/) || (RUBY_PLATFORM =~ /mswin|mingw/))
+
+if ENV["SDM_ADMIN_TOKEN"]
+  puts('Starting SDM')
+  result = system("sdm --admin-token #{ENV['SDM_ADMIN_TOKEN']} login")
+  unless result
+    puts "You need to install SDM CLI."
+    return
+  end
+  listen_thread = Thread.new { system('sdm listen', :out => File::NULL, :err => File::NULL) }
+  until system('sdm status', :out => File::NULL) do
+    sleep(1)
+  end
+end
+
+puts("Creating Fluentd conf file")
+require "#{ENV['FLUENTD_DIR']}/../create-conf.rb"
+
+fluentd_pkg_name = "fluentd"
+fluentd_pkg_version = "> 0.a"
+
+puts("Starting Fluentd")
+
+if ::USING_WINDOWS
+  fluentd_path = "fluentd"
+elsif Gem.respond_to?(:activate_bin_path)
+  # when using Linux sometimes the "fluentd" binary is not called,
+  # so to ensure the proper execution we need to provide the full binary path
+  fluentd_path = Gem.activate_bin_path(fluentd_pkg_name, fluentd_pkg_name, fluentd_pkg_version)
+else
+  fluentd_path = Gem.bin_path(fluentd_pkg_name, fluentd_pkg_name, fluentd_pkg_version)
+end
+
+system("#{fluentd_path} -c #{ENV['FLUENTD_DIR']}/etc/fluent.conf -p #{ENV['FLUENTD_DIR']}/plugins")

metadata

@@ -0,0 +1,365 @@
+--- !ruby/object:Gem::Specification
+name: log-export-container
+version: !ruby/object:Gem::Version
+  version: 1.0.53
+platform: ruby
+authors:
+- StrongDM
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-10-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-rewrite-tag-filter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-cloudwatch-logs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-splunk-hec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-datadog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-azure-loganalytics
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-sumologic_output
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-sanitizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-logzio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-grafana-loki
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-remote_syslog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.2.4
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-plugin-prometheus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: The application acts as a syslog concentrator. Customers that want to
+  export their strongDM query logs to a third party logging service can use the application
+  to do so. They configure the application for the appropriate target. Deploy the
+  application. Configure their strongDM gateways to logs to a syslog destination and
+  set the destination to the address of the logging application host.
+email: 
+executables:
+- log-export-container
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/log-export-container
+- conf-utils.rb
+- create-conf.rb
+- fluentd/etc/classify-default-csv.conf
+- fluentd/etc/classify-default-json.conf
+- fluentd/etc/classify-syslog-csv.conf
+- fluentd/etc/classify-tcp-csv.conf
+- fluentd/etc/input-extract-audit-entities.conf
+- fluentd/etc/input-file-csv.conf
+- fluentd/etc/input-file-json.conf
+- fluentd/etc/input-json-chunk.conf
+- fluentd/etc/input-syslog-csv.conf
+- fluentd/etc/input-syslog-json.conf
+- fluentd/etc/input-tcp-csv.conf
+- fluentd/etc/input-tcp-json.conf
+- fluentd/etc/monitoring.conf
+- fluentd/etc/output-azure-loganalytics.conf
+- fluentd/etc/output-bigquery.conf
+- fluentd/etc/output-cloudwatch.conf
+- fluentd/etc/output-datadog.conf
+- fluentd/etc/output-elasticsearch-8.conf
+- fluentd/etc/output-kafka.conf
+- fluentd/etc/output-logz.conf
+- fluentd/etc/output-loki.conf
+- fluentd/etc/output-mongo.conf
+- fluentd/etc/output-remote-syslog.conf
+- fluentd/etc/output-s3.conf
+- fluentd/etc/output-splunk-hec.conf
+- fluentd/etc/output-stdout.conf
+- fluentd/etc/output-sumologic.conf
+- fluentd/etc/output-template.conf
+- fluentd/etc/process.conf
+- fluentd/plugins/filter_sdm_decode_chunk_events.rb
+- fluentd/plugins/parser_sdm_json.rb
+- fluentd/scripts/dump_sdm_entities.rb
+- start.rb
+homepage: https://strongdm.github.io/log-export-container/
+licenses:
+- Apache-2.0
+metadata:
+  source_code_uri: https://github.com/strongdm/log-export-container
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key: 
+specification_version: 4
+summary: An application that can be easily deployed and configured to export strongDM
+  query logs
+test_files: []