dmarcurator 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4db33b8efc1496dabb33d6612b082a56c47a1b28
+  data.tar.gz: 0a60be2fbc7fcc09ae27c6d4fcee25cbd1eb0fd9
+SHA512:
+  metadata.gz: 25fd8f9c7898fc133c58b27645921e01ab085a8a86dc319618ddd110cee1f555ceee3067b01fd5c9436c4d53fae0d726a07acabd1e6d44c06519be99848cae2e
+  data.tar.gz: 908dc64c6d499f5e37e5068c1cc5270154ea7114e83b1ca2c10d226809232cfe3ca4cabfa21bce1590c56731af7411bf7548751b78489352d056643c9333f395

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+*.lock
+.bundle
+/tmp

data/.ruby-version

@@ -0,0 +1 @@
+2.3.1

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+source "https://rubygems.org"
+
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Ayumi Yu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,21 @@
+## dmarcurator
+
+Imports DMARC report XML into SQLite DB.
+
+- [More info on email authentication](https://jl.ly/Email/authcheat.html)
+
+## install
+
+Install sqlite3 bc it's cool and you need it
+`brew install sqlite3`
+
+`gem install dmarcurator`
+
+## usage
+
+- Get some DMARC reports in XML and put them in a directory.
+- `dmarcurator --db=./dmarc/reports.sqlite --reports-path=./dmarc/reports`
+
+## how to browse db
+
+[SQLite-web](https://github.com/coleifer/sqlite-web) is a deec interface.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/dmarcurator

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require "dmarcurator"
+Dmarcurator::Cli::App.main

data/dmarcurator.gemspec

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+require File.expand_path("../lib/dmarcurator/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.add_dependency "ox", "~> 2.4"
+  s.add_dependency "sequel", "~> 4.38"
+  s.add_dependency "sqlite3", "~> 1.3"
+  s.add_development_dependency "bundler", "~> 1.13"
+  s.add_development_dependency "pry", "~> 0"
+  s.add_development_dependency "rake", "~> 0"
+  s.authors = ["Ayumi Yu"]
+  s.description = "Simple tool for viewing DMARC reports"
+  s.email = ["ayumi@ayumiyu.com"]
+  s.files = `git ls-files`.split("\n")
+  s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.homepage = "https://github.com/ayumi/dmarcurator"
+  s.license = "MIT"
+  s.name = "dmarcurator"
+  s.summary = "Tool for viewing DMARC reports"
+  s.require_paths = ["lib"]
+  s.version = Dmarcurator::VERSION
+end

data/lib/dmarcurator/cli/app.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+module Dmarcurator
+  module Cli
+    class App
+      require "pry"
+
+      attr_reader :db_uri, :reports_path
+
+      def self.main
+        params = parse_options(ARGV)
+        new(params).run
+      end
+
+      def self.parse_options(options)
+        params = {}
+
+        opt_parser = OptionParser.new do |parser|
+          parser.banner = "dmarcurator parses DMARC reports and stores them into an SQLite3 DB.\nIt can also serve a basic web UI for viewing reports (-ui=true).\nUsage: dmarcurator [options]"
+
+          parser.on("-db", "--db-sqlite=SQLITE_DB", "[Required] Path to sqlite3 DB. (e.g. ./tmp/reports.sqlite)") do |value|
+            params[:db_path] = value
+          end
+
+          parser.on("-rp", "--reports-path=REPORTS_PATH", "Path to directory containing DMARC reports. (e.g. ./tmp/reports/)") do |value|
+            params[:reports_path] = value
+          end
+
+          parser.on("-h", "--help", "Halp pls") do
+            puts parser
+            exit 0
+          end
+
+          parser.on("-v", "--version", "Print version") do
+            puts ::Dmarcurator::VERSION
+            exit 0
+          end
+
+          if options.empty?
+            puts parser
+            exit 1
+          end
+        end
+
+        opt_parser.parse!(options)
+        if !params[:reports_path]
+          puts "Dmarcurator can import DMARC reports into an sqlite3 database -> Set --db and --reports-path)"
+          exit 0
+        end
+
+        params
+      end
+
+      def initialize(db_path:, reports_path: nil)
+        @db_uri = "sqlite://#{File.expand_path(db_path)}"
+        @reports_path = reports_path
+      end
+
+      def run
+        if reports_path
+          ::Dmarcurator::ImportReports.new(db_uri: db_uri, reports_path: reports_path).run
+        end
+      end
+    end
+  end
+end

data/lib/dmarcurator/import_reports.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+require "sequel"
+
+module Dmarcurator
+  # Import DMARC XML reports into DB
+  class ImportReports
+    attr_reader :db, :reports_path
+
+    def initialize(db_uri:, reports_path:)
+      @db = Sequel.connect(db_uri)
+      @reports_path = reports_path
+    end
+
+    def run
+      puts "Importing #{reports_path}"
+      Dir.foreach(reports_path) do |path|
+        next if path == '.' || path == '..' || File.extname(path) != ".xml"
+        puts "  #{path}"
+        parsed_report = ::Dmarcurator::Parser::Report.new(xml: "#{reports_path}/#{path}")
+        ::Dmarcurator::Store::Report.import_parsed(db: db, parsed: parsed_report)
+      end
+      puts "Done importing :)"
+    end
+  end
+end

data/lib/dmarcurator/parser/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+require "open3"
+require "ox"
+
+module Dmarcurator
+  module Parser
+    # Base XML parser
+    class Base
+      attr_reader :doc
+
+      def initialize(xml: nil, parsed_xml: nil)
+        if xml
+          content = File.read(xml)
+          @doc = Ox.parse(content)
+        elsif parsed_xml
+          @doc = parsed_xml
+        else
+          raise "Either :xml or :parsed_xml are required"
+        end
+      end
+    end
+  end
+end

data/lib/dmarcurator/parser/record.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+module Dmarcurator
+  module Parser
+    # Parsed XML of a record
+    class Record < Base
+      def source_ip
+        doc.locate("row/source_ip")[0].text
+      end
+
+      def count
+        doc.locate("row/count")[0].text.to_i
+      end
+
+      def disposition
+        doc.locate("row/policy_evaluated/disposition")[0].text
+      end
+
+      def policy_result_dkim
+        doc.locate("row/policy_evaluated/dkim")[0].text
+      end
+
+      def policy_result_spf
+        doc.locate("row/policy_evaluated/spf")[0].text
+      end
+
+      def envelope_to
+        doc.locate("identifiers/envelope_to")[0]&.text
+      end
+
+      def header_from
+        doc.locate("identifiers/header_from")[0].text
+      end
+
+      def auth_dkim_domain
+        doc.locate("auth_results/dkim/domain")[0]&.text
+      end
+
+      def auth_dkim_result
+        doc.locate("auth_results/dkim/result")[0]&.text
+      end
+
+      def auth_dkim_selector
+        doc.locate("auth_results/dkim/selector")[0]&.text
+      end
+
+      def auth_spf_domain
+        doc.locate("auth_results/spf/domain")[0].text
+      end
+
+      def auth_spf_result
+        doc.locate("auth_results/spf/result")[0].text
+      end
+    end
+  end
+end

data/lib/dmarcurator/parser/report.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+module Dmarcurator
+  module Parser
+    # Parsed XML of a report
+    class Report < Base
+      def org_name
+        doc.locate("feedback/report_metadata/org_name")[0].text
+      end
+
+      def email
+        doc.locate("feedback/report_metadata/email")[0].text
+      end
+
+      def extra_contact_info
+        doc.locate("feedback/report_metadata/extra_contact_info")[0]&.text
+      end
+
+      def dmarc_report_id
+        doc.locate("feedback/report_metadata/report_id")[0].text
+      end
+
+      def error
+        doc.locate("feedback/report_metadata/error")[0]&.text
+      end
+
+      def begin_at
+        Time.at(doc.locate("feedback/report_metadata/date_range/begin")[0].text.to_i)
+      end
+
+      def end_at
+        Time.at(doc.locate("feedback/report_metadata/date_range/end")[0].text.to_i)
+      end
+
+      def policy_domain
+        doc.locate("feedback/policy_published/domain")[0].text
+      end
+
+      def policy_adkim
+        doc.locate("feedback/policy_published/adkim")[0]&.text
+      end
+
+      def policy_aspf
+        doc.locate("feedback/policy_published/aspf")[0]&.text
+      end
+
+      def policy_p
+        doc.locate("feedback/policy_published/p")[0].text
+      end
+
+      def policy_sp
+        doc.locate("feedback/policy_published/sp")[0]&.text
+      end
+
+      def policy_pct
+        doc.locate("feedback/policy_published/pct")[0].text.to_i
+      end
+
+      def records
+        doc.locate("feedback/record").map do |record|
+          Record.new(parsed_xml: record)
+        end
+      end
+    end
+  end
+end

data/lib/dmarcurator/parser.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# XML report parsers
+module Dmarcurator
+  module Parser
+  end
+end

data/lib/dmarcurator/store/record.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require "sequel"
+
+module Dmarcurator
+  class Store
+    # DMARC report. Contains many records.
+    class Record
+      attr_reader :db
+
+      def self.create_table(db:)
+        db.create_table :records do
+          primary_key :id
+          foreign_key(:report_id, :reports, key: :id)
+          String :source_ip
+          Integer :count
+          String :disposition
+          String :policy_result_dkim
+          String :policy_result_spf
+          String :envelope_to
+          String :header_from
+          String :auth_dkim_domain
+          String :auth_dkim_result
+          String :auth_dkim_selector
+          String :auth_spf_domain
+          String :auth_spf_result
+        end
+      end
+
+      def self.import_parsed(db:, parsed:, report_id:)
+        create_table(db: db) if !db.table_exists?(:records)
+
+        attributes = { report_id: report_id }
+        db[:records].columns.each do |attribute|
+          next if !parsed.respond_to?(attribute)
+          attributes[attribute] = parsed.public_send(attribute)
+        end
+        result = db[:records].insert(attributes)
+        STDOUT << "."
+      end
+    end
+  end
+end

data/lib/dmarcurator/store/report.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+# DMARC report. Contains many records.
+module Dmarcurator
+  class Store
+    class Report
+      attr_reader :db
+
+      def self.create_table(db:)
+        puts "Creating table: reports"
+        db.create_table :reports do
+          primary_key :id
+          String :org_name
+          String :email
+          String :extra_contact_info
+          String :dmarc_report_id
+          String :error
+          DateTime :begin_at
+          DateTime :end_at
+          String :policy_domain
+          String :policy_adkim
+          String :policy_aspf
+          String :policy_p
+          String :policy_sp
+          Integer :policy_pct
+
+          index :dmarc_report_id
+        end
+      end
+
+      def self.import_parsed(db:, parsed:)
+        create_table(db: db) if !db.table_exists?(:reports)
+        if db[:reports].where(dmarc_report_id: parsed.dmarc_report_id).count > 0
+          puts "Report exists; skipping"
+          return
+        end
+
+        attributes = {}
+        db[:reports].columns.each do |attribute|
+          next if !parsed.respond_to?(attribute)
+          attributes[attribute] = parsed.public_send(attribute)
+        end
+        result = db[:reports].insert(attributes)
+        imported_report_id = db[:reports].select(:id).order('id ASC').limit(1).first[:id]
+
+        parsed.records.each do |parsed_record|
+          Record.import_parsed(db: db, parsed: parsed_record, report_id: imported_report_id)
+        end
+        puts "Report++"
+      end
+    end
+  end
+end

data/lib/dmarcurator/store.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# SQL Store for reports
+module Dmarcurator
+  class Store
+  end
+end

data/lib/dmarcurator/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Dmarcurator
+  VERSION = "0.1.1"
+end

data/lib/dmarcurator.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+require "dmarcurator/version"
+
+module Dmarcurator
+  require "dmarcurator/cli/app"
+  require "dmarcurator/import_reports"
+  require "dmarcurator/parser"
+  require "dmarcurator/parser/base"
+  require "dmarcurator/parser/record"
+  require "dmarcurator/parser/report"
+  require "dmarcurator/store"
+  require "dmarcurator/store/record"
+  require "dmarcurator/store/report"
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: dmarcurator
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Ayumi Yu
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ox
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.38'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.38'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple tool for viewing DMARC reports
+email:
+- ayumi@ayumiyu.com
+executables:
+- dmarcurator
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/dmarcurator
+- dmarcurator.gemspec
+- lib/dmarcurator.rb
+- lib/dmarcurator/cli/app.rb
+- lib/dmarcurator/import_reports.rb
+- lib/dmarcurator/parser.rb
+- lib/dmarcurator/parser/base.rb
+- lib/dmarcurator/parser/record.rb
+- lib/dmarcurator/parser/report.rb
+- lib/dmarcurator/store.rb
+- lib/dmarcurator/store/record.rb
+- lib/dmarcurator/store/report.rb
+- lib/dmarcurator/version.rb
+homepage: https://github.com/ayumi/dmarcurator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Tool for viewing DMARC reports
+test_files: []