siteguard_lite-log-parser 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 201cdadcf28bec2479fd6d3061b6a9029ffccf9d87c26052340084f008f3ffd3
+  data.tar.gz: b8084f5ae96fe77ff97d9086aa50ebfb4a978f549a3eb7e811c924f383f0bfa9
+SHA512:
+  metadata.gz: 51e3881e30fbc03048081b002f71ee7c4ed009aebb451f8f6ebe905d593dcc2b5f64b491666d6065997a3ebd4cee1185aa80c191b1cba3a6b8d114b15362869d
+  data.tar.gz: '0988616d61a22ded28f65ec9d3a270337db73e6291fc6791de62f70d18966534ad7df914564c1bf187001c5d1542c92ec421ff3bc0da140cd45cffad7bb7b985'

data/.gitignore

@@ -0,0 +1,12 @@
+/Gemfile.lock
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.16.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in siteguard_lite-log-parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 GMO Pepabo, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,106 @@
+# SiteguardLite::Log::Parser
+
+A log parser for SiteGuard Lite WAF.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'siteguard_lite-log-parser'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install siteguard_lite-log-parser
+
+## Usage
+
+```ruby
+require 'siteguard_lite/log/parser'
+
+parser = SiteguardLiteLogParser.new(log_type)
+log = parser.parse(log_str)
+```
+
+The supported log types are following.
+
+- `detect`: Parse a `detect.log` format string
+
+The `new` function accepts a optional hash with following keys.
+
+- `leading_time`: A time string at the beginning of the line (optional, default: `false`)
+  - SiteGuard Liteの管理画面からダウンロードしたログファイルには行頭に`YYYY-MM-DD hh:mm:ss : `という形式の日時文字列が付いているので、このオプションが必要です。
+
+### detect.log
+
+The parse result have the following keys.
+
+- `time`: クライアントから接続された時刻です。`YYYY-MM-DD hh:mm:ss`形式で表示します。(optional)
+- `time_epoch`: クライアントから接続された時刻です。エポックタイム (1970/01/01 00:00:00(UTC)) からの秒数をミリ秒単位で表示します。
+- `conn_time`: クライアントとの接続時間をミリ秒単位で表示します。
+- `client_ip`: クライアントの IP アドレスを表示します。
+- `file_size`: 転送したファイルのサイズです。
+- `http_method`: HTTP の要求メソッド (GET, POST 等) です。
+- `url`: 接続先の URL です。
+- `hierarchy_code`: "DIRECT/本製品をインストールしているサーバーの IP アドレス" を表示します。
+- `content_type`: 送受信するファイルの Content-Type を表示します。利用できない場合は "-" となります。
+- `detect_stat`: 検出情報。`DETECT-STAT:WAF:[detect_name]::[detect_str]:[detect_str_all]:`を表示します。
+- `detect_name`: 検出名。以下のいずれかです。
+  - シグネチャ検査: `RULE_SIG/[rule_sig_part]/[rule_sig_name]/[rule_sig_file]/[rule_sig_id]/[rule_sig_signature_name]`
+    - `detect_name_rule`: RULE_SIG
+    - `rule_sig_part`: 検出箇所
+    - `rule_sig_name`: 名前。パラメータ変数、ヘッダフィールド名を表示します。
+    - `rule_sig_file`: シグネチャファイル。OFFICIAL(トラステッド・シグネチャ)、CUSTOM(カスタム・シグネチャ) のいずれかです。
+    - `rule_sig_id`: シグネチャID
+    - `rule_sig_signature_name`: シグネチャ名
+  - フィルタ: `WAF_FILTER/[IP アドレス]` **(NOT SUPPORTED)**
+  - URL デコードエラー: `RULE_URLDECODE/[検出箇所]/[名前]` **(NOT SUPPORTED)**
+  - パラメータ数の上限値の制限: `RULE_PARAMS_NUM/[rule_params_num_part/[rule_params_num_threshold]`
+    - `detect_name_rule`: RULE_PARAMS_NUM
+    - `rule_params_num_part`: 検出箇所
+    - `rule_params_num_threshold`: パラメータ数の上限値
+- `detect_str`: 検出文字列
+- `detect_str_all`: 検出文字列(全体)
+- `action`: 動作。`ACTION:[action_str]:`を表示します。
+- `action_str`: 動作。MONITOR, BLOCK, FILTERのいずれかです。
+- `judge`: `JUDGE:[judge_str]:[monitor_url]:`を表示します。
+- `judge_str`: 判定。MONITOR, BLOCK, FILTERのいずれかです。
+- `monitor_url`: 監視 URL の設定。0(監視 URL に該当しない)、1(監視 URL に該当する)のいずれかです。
+- `search_key`: 検索キー。`SEARCH-KEY:[search_key_time_epock.seach_key_nginx_request_id]:`を表示します。
+- `search_key_time_epoch`: 時刻(エポックタイム)
+- `search_key_nginx_request_id`: nginx リクエスト ID
+
+## siteguard_lite-log
+
+The command line tool to parse logs. This tool output as LTSV format.
+
+```
+cat detect.log | siteguard_lite-log
+```
+
+Usage:
+```
+$ siteguard_lite-log --help
+Usage: siteguard_lite-log [options]
+        --type VAL                   Specify log type. (default: detect)
+        --leading-time               The log have the time string at heading of the line
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/pepabo/siteguard_lite-log-parser.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "siteguard_lite/log/parser"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/siteguard_lite-log

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'siteguard_lite/log/cli'
+require 'siteguard_lite/log/parser'
+
+options = {
+  type: 'detect',
+  format: 'ltsv',
+  leading_time: false,
+}
+
+opt = OptionParser.new
+opt.on('--type VAL', 'Specify log type. (default: detect)') { |v|
+  options[:type] = v
+}
+opt.on('--leading-time', 'The log have the time string at heading of the line') { |v|
+  options[:leading_time] = true
+}
+opt.parse!(ARGV)
+
+SiteguardLite::Log::CLI.new(options).run

data/lib/siteguard_lite/log/cli.rb

@@ -0,0 +1,31 @@
+module SiteguardLite
+  module Log
+    class CLI
+      def initialize(options)
+        @type = options.delete(:type)
+        @format = options.delete(:format)
+        @parser = SiteguardLiteLogParser.new(@type, options)
+      end
+
+      def run
+        while line = STDIN.gets
+          line.chomp!
+          result = @parser.parse(line)
+          puts format(result)
+        end
+      end
+
+      private
+
+      def format(h)
+        case @format
+        when 'ltsv'
+          require 'ltsv'
+          LTSV.dump(h)
+        else
+          raise ArgumentError, "Unexpected output format: #{@format}"
+        end
+      end
+    end
+  end
+end

data/lib/siteguard_lite/log/detect.rb

@@ -0,0 +1,69 @@
+module SiteguardLite
+  module Log
+    class Detect
+      TIME = '(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})'.freeze
+      TIME_EPOCH = '(?<time_epoch>\d+\.\d+)'.freeze
+      CONN_TIME = '(?<conn_time>\d+)'.freeze
+      CLIENT_IP = '(?<client_ip>[\da-f.:]+)'.freeze
+      RESULT = 'TCP_MISS\/000'.freeze
+      FILE_SIZE = '(?<file_size>\d+)'.freeze
+      HTTP_METHOD = '(?<http_method>[A-Z]+)'.freeze
+      URL = '(?<url>[^\s]+)'.freeze
+      USER = '-'.freeze
+      HIERARCHY_CODE = '(?<hierarchy_code>[^\s]+)'.freeze
+      CONTENT_TYPE = '(?<content_type>[^\s]+)'.freeze
+
+      RULE_SIG = '(?<detect_name_rule>RULE_SIG)\/(?<rule_sig_part>[^\/]+)\/(?<rule_sig_name>[^\/]+)\/(?<rule_sig_file>(?:OFFICIAL|CUSTOM))\/(?<rule_sig_id>[^\/]+)\/(?<rule_sig_signature_name>[\w\d-]+)'.freeze
+      WAF_FILTER = '(?<detect_name_rule>WAF_FILTER)'.freeze
+      RULE_URLDECODE = '(?<detect_name_rule>RULE_URLDECODE)'.freeze
+      RULE_PARAMS_NUM = '(?<detect_name_rule>RULE_PARAMS_NUM)\/(?<rule_params_num_part>[^\/]+)\/(?<rule_params_num_threshold>\d+)'.freeze
+      DETECT_NAME = "(?<detect_name>(?:#{RULE_SIG}|#{WAF_FILTER}|#{RULE_URLDECODE}|#{RULE_PARAMS_NUM}))".freeze
+      DETECT_STAT = "(?<detect_stat>DETECT-STAT:WAF:#{DETECT_NAME}::(?<detect_str>[^:]*):(?<detect_str_all>[^:]+):)".freeze
+
+      ACTION = '(?<action>ACTION:(?<action_str>[A-Z]+):)'.freeze
+      JUDGE = '(?<judge>JUDGE:(?<judge_str>[A-Z]+):(?<monitor_url>0|1):)'.freeze
+      SEARCH_KEY = '(?<search_key>SEARCH-KEY:(?<search_key_time_epoch>\d+\.\d+)\.(?<search_key_nginx_request_id>[^:]+):)'.freeze
+
+      def initialize(leading_time: false)
+        @leading_time = leading_time
+      end
+
+      def parse(line_str)
+        if m = line_str.match(pattern)
+          m.named_captures
+        else
+          {}
+        end
+      end
+
+      private
+
+      def pattern
+        @pattern ||= if @leading_time
+          /\A#{TIME} : #{pattern_parts.join('\s+')}/
+        else
+          /\A#{pattern_parts.join('\s+')}/
+        end
+      end
+
+      def pattern_parts
+        [
+          TIME_EPOCH,
+          CONN_TIME,
+          CLIENT_IP,
+          RESULT,
+          FILE_SIZE,
+          HTTP_METHOD,
+          URL,
+          USER,
+          HIERARCHY_CODE,
+          CONTENT_TYPE,
+          DETECT_STAT,
+          ACTION,
+          JUDGE,
+          SEARCH_KEY,
+        ]
+      end
+    end
+  end
+end

data/lib/siteguard_lite/log/parser.rb

@@ -0,0 +1,10 @@
+require "siteguard_lite/log/parser/version"
+require "siteguard_lite_log_parser"
+
+module SiteguardLite
+  module Log
+    module Parser
+      # Your code goes here...
+    end
+  end
+end

data/lib/siteguard_lite/log/parser/version.rb

@@ -0,0 +1,7 @@
+module SiteguardLite
+  module Log
+    module Parser
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/siteguard_lite_log_parser.rb

@@ -0,0 +1,29 @@
+require 'siteguard_lite/log/detect'
+
+class SiteguardLiteLogParser
+  def initialize(type, options = {})
+    @type = type
+    @options = options
+
+    @parser = get_parser(type)
+  end
+
+  def parse(line_str)
+    @parser.parse(line_str)
+  end
+
+  private
+
+  def get_parser(type)
+    case type
+    when 'detect'
+      SiteguardLite::Log::Detect.new(leading_time: leading_time)
+    else
+      raise ArgumentError, "Unexpected log type: #{type}"
+    end
+  end
+
+  def leading_time
+    @options[:leading_time]
+  end
+end

data/siteguard_lite-log-parser.gemspec

@@ -0,0 +1,29 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "siteguard_lite/log/parser/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "siteguard_lite-log-parser"
+  spec.version       = SiteguardLite::Log::Parser::VERSION
+  spec.authors       = ["Takatoshi Ono"]
+  spec.email         = ["takatoshi.ono@gmail.com"]
+
+  spec.summary       = %q{A log parser for SiteGuard Lite WAF}
+  spec.description   = %q{A log parser for SiteGuard Lite WAF}
+  spec.homepage      = "https://github.com/pepabo/siteguard_lite-log-parser"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "ltsv"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: siteguard_lite-log-parser
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Takatoshi Ono
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ltsv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A log parser for SiteGuard Lite WAF
+email:
+- takatoshi.ono@gmail.com
+executables:
+- siteguard_lite-log
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/siteguard_lite-log
+- lib/siteguard_lite/log/cli.rb
+- lib/siteguard_lite/log/detect.rb
+- lib/siteguard_lite/log/parser.rb
+- lib/siteguard_lite/log/parser/version.rb
+- lib/siteguard_lite_log_parser.rb
+- siteguard_lite-log-parser.gemspec
+homepage: https://github.com/pepabo/siteguard_lite-log-parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: A log parser for SiteGuard Lite WAF
+test_files: []