fluent-plugin-record-sort 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a597fc13e52932c701e620431cafd05bdc5b78b9f89d7b78dbd35adbb4427c00
+  data.tar.gz: f27104cea5600636f4fcb70fe52a5a8776aa1469481ed059759abb68a43adbd7
+SHA512:
+  metadata.gz: 15d3b4b87ceba36c922c8861adcc6e3faa42f534b9978bced01bb7c6aecc345041b2f7cdfbe4349420259486a02357a7002f31c85733657d125c575332283b0e
+  data.tar.gz: 7aaa2eba49f30fcdbe8a38910ebe6d4668b89b989a0cedbd5ab3c0a324a19cfca55c2e4c2565df818e351e8c78f6e9c3408000a9c7c2ab13373629621f2a77e7

data/.circleci/config.yml

@@ -0,0 +1,70 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.3
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      - run:
+          name: run default rake target
+          command: |
+            bundle exec rake
+
+      - run:
+          name: publish to rubygems.org
+          command: |
+            bin/deploy
+
+      - run:
+          name: list filesystem
+          command: find .
+
+      # run tests!
+#      - run:
+#          name: run tests
+#          command: |
+#            mkdir /tmp/test-results
+#            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+#
+#            bundle exec rspec --format progress \
+#                            --format RspecJunitFormatter \
+#                            --out /tmp/test-results/rspec.xml \
+#                            --format progress \
+#                            $TEST_FILES
+#
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: pkg

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+
+# We're a library
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,4 @@
+AllCops:
+  Exclude:
+    - fluent-plugin-record-sort.gemspec
+    - vendor/**/*

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.16.3

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in fluent-plugin-record-sort.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Sam Pointer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,17 @@
+# Fluentd Sort Plugin [![CircleCI](https://circleci.com/gh/sampointer/fluent-plugin-record-sort.svg?style=shield)](https://circleci.com/gh/sampointer/fluent-plugin-record-sort)
+
+## Introduction
+This plugin does two things:
+
+* Sorts the field in each record according to the default ruby `#Array.sort`
+* Removes fields who's values it considers to either be `nil?` or `empty?`
+
+## Configuration
+```xml
+<filter **>
+  @type record_sort
+</filter>
+```
+
+## Rationale
+AWS Athena and Glue prefer sorted JSON in S3 that has truthy values for all keys.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: %i[rubocop spec build]

data/assets/credentials

@@ -0,0 +1,2 @@
+---
+:rubygems_api_key: REPLACEME

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'fluent/plugin/record/sort'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/deploy

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# deploy to rubygems.
+
+if [[ -z ${CI} ]]; then
+  echo "not running inside a CI system, exiting"
+  exit 1
+fi
+
+gem=$(ls pkg/*.gem | tail -1)
+mkdir -p ~/.gem/
+cp assets/credentials ~/.gem/credentials
+sed -i "s/REPLACEME/${rubygems_api_key}/g" ~/.gem/credentials
+chmod 0600 ~/.gem/credentials
+echo "Pushing ${gem}"
+gem push ${gem}

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/fluent-plugin-record-sort.gemspec

@@ -0,0 +1,41 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'fluent/plugin/record/sort/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'fluent-plugin-record-sort'
+  spec.version       = Fluent::Plugin::Record::Sort::VERSION
+  spec.authors       = ['Sam Pointer']
+  spec.email         = ['sam@outsidethe.net']
+  spec.summary       = 'Fluentd plugin for sorting record fields.'
+  spec.description   = 'Fluentd plugin for sorting record fields.'
+  spec.homepage      = 'https://github.com/sampointer/fluent-plugin-record-sort'
+  spec.license       = 'MIT'
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the
+  # 'allowed_push_host' # to allow pushing to a single host or delete this
+  # section to allow pushing to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against ' \
+      'public gem pushes.'
+  end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added
+  # into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features)/})
+    end
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '0.58.2'
+end

data/lib/fluent/plugin/filter_record_sort.rb

@@ -0,0 +1,30 @@
+require 'fluent/plugin/filter'
+require 'fluent/config/error'
+
+module Fluent
+  module Plugin
+    #
+    # Filter to sort records and remove fields with no value
+    #
+    class RecordSortFilter < Fluent::Plugin::Filter
+      Fluent::Plugin.register_filter('record_sort', self)
+      def configure(conf)
+        super
+      end
+
+      def filter(_tag, _time, record)
+        # Remove empty fields
+        begin
+          record.reject! do |_k, v|
+            v.nil? || (v.respond_to?(:empty?) && v.empty?)
+          end
+        rescue NoMethodError
+          log.warn('record_sort plugin has rescued NoMethodError, meaning it'\
+                    ' let an object pass through without empty field filtering')
+        end
+
+        record.sort.to_h
+      end
+    end
+  end
+end

data/lib/fluent/plugin/record/sort.rb

@@ -0,0 +1,12 @@
+require 'fluent/plugin/record/sort/version'
+
+module Fluent
+  module Plugin
+    module Record
+      # Container
+      module Sort
+        # Your code goes here...
+      end
+    end
+  end
+end

data/lib/fluent/plugin/record/sort/version.rb

@@ -0,0 +1,9 @@
+module Fluent
+  module Plugin
+    module Record
+      module Sort
+        VERSION = '0.1.0'.freeze
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-record-sort
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sam Pointer
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-08-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.58.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.58.2
+description: Fluentd plugin for sorting record fields.
+email:
+- sam@outsidethe.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- assets/credentials
+- bin/console
+- bin/deploy
+- bin/setup
+- fluent-plugin-record-sort.gemspec
+- lib/fluent/plugin/filter_record_sort.rb
+- lib/fluent/plugin/record/sort.rb
+- lib/fluent/plugin/record/sort/version.rb
+homepage: https://github.com/sampointer/fluent-plugin-record-sort
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.7
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin for sorting record fields.
+test_files: []