mbox_header_scraper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7be5524c12492b10eb63bfad40bac97efbce5ca3
+  data.tar.gz: 482a7926ad5e837d475b7615c3c60ee029b790bd
+SHA512:
+  metadata.gz: 07e78ca647f47a93066b03bcceaa2dbb9a357103313ee780a74fc31d2f5fef5ae844073b1a58998476b162a98cf9839cc8cb26797daaa6c5d2583c0e89d6c3fa
+  data.tar.gz: 7dbd55635c648fe3feab7b730ec38f8db04a0339b49d323c3531d3c9a5d766c8c888f3daf7f5ebfbef1e1f3b7a25f098bc1cddd756c2f9ca84694b7ab8916aa9

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/bundle

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,61 @@
+# This configuration was generated by `rubocop --auto-gen-config`
+# on 2014-09-11 18:44:31 +0900 using RuboCop version 0.26.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+AllCops:
+  Exclude:
+    - 'bin/**/*'
+    - 'db/**/*'
+    - 'config/**/*'
+    - 'script/**/*'
+    - 'vendor/**/*'
+    - 'spec/spec_helper.rb'
+    - 'spec/rails_helper.rb'
+    - 'Guardfile'
+    #- !ruby/regexp /old_and_unused\.rb$/
+
+# Configuration parameters: CountComments.
+Metrics/ClassLength:
+  Max: 500
+
+# Configuration parameters: AllowURI.
+Metrics/LineLength:
+  Max: 120
+
+# Configuration parameters: CountComments.
+Metrics/MethodLength:
+  Max: 30
+
+Metrics/AbcSize:
+  Enabled: true
+  Max: 17
+
+Metrics/CyclomaticComplexity:
+  Enabled: true
+  Max: 10
+
+Style/AsciiComments:
+  Enabled: false
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/BracesAroundHashParameters:
+  Enabled: false
+
+# Cop supports --auto-correct.
+#Style/EmptyLinesAroundBody:
+#  Enabled: false
+
+# Cop supports --auto-correct.
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+Style/SignalException:
+  Enabled: false
+
+Style/WordArray:
+  MinSize: 10
+
+ClassAndModuleChildren:
+  EnforcedStyle: compact

data/.ruby-version

@@ -0,0 +1 @@
+2.0.0-p0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mbox_header_scraper.gemspec
+gemspec

data/README.md

@@ -0,0 +1,56 @@
+# MboxHeaderScraper
+
+[![Circle CI](https://circleci.com/gh/tamano/mbox_header_scraper.svg?style=svg)](https://circleci.com/gh/tamano/mbox_header_scraper) [![Code Climate](https://codeclimate.com/github/tamano/mbox_header_scraper/badges/gpa.svg)](https://codeclimate.com/github/tamano/mbox_header_scraper) [![Test Coverage](https://codeclimate.com/github/tamano/mbox_header_scraper/badges/coverage.svg)](https://codeclimate.com/github/tamano/mbox_header_scraper)
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/mbox_header_scraper`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'mbox_header_scraper'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mbox_header_scraper
+
+## Usage
+
+How to use.
+
+```bash
+$ mbox_header_scraper mail_address_list {mbox_file} {output_tsv}
+```
+
+Then, result TSV file goes like this.
+
+```
+Subject	Date	From	To	CC
+My Subject	Fri, 15 May 2015 18:14:47 +0900	from@example.com	to1@example.com	
+			to2@example.com	
+テスト	Thu, 14 May 2015 16:39:43 +0900	from@example.com	to@example.com	cc@example.com
+```
+
+That's all!
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/mbox_header_scraper/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mbox_header_scraper"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start
+
+# require "irb"
+# IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/circle.yml

@@ -0,0 +1,11 @@
+machine:
+  timezone:
+    Asia/Tokyo
+
+dependencies:
+  pre:
+    - gem install bundler --pre
+
+test:
+  pre:
+    - bundle exec rubocop

data/exe/mbox_header_scraper

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'mbox_header_scraper'
+
+MboxHeaderScraper::CLI.start

data/lib/mbox_header_scraper/cli.rb

@@ -0,0 +1,28 @@
+# Command Interface
+class MboxHeaderScraper::CLI < Thor
+  desc \
+    'mail_address_list {in file} {out file}',
+    'load {in file} and output email address included to {out file}'
+  long_desc \
+    'load mbox file and output in tsv format file. ' \
+    'the file includes No, Subject(short), From, To, CC'
+  def mail_address_list(in_file, out_file)
+    error_message = MboxHeaderScraper::Scraper.check_in_file(in_file)
+    if error_message
+      puts "input file error: #{error_message}"
+      exit 1
+    end
+
+    error_message = MboxHeaderScraper::Scraper.check_out_file(out_file)
+    if error_message
+      puts "output file error: #{error_message}"
+      exit 1
+    end
+
+    MboxHeaderScraper::Scraper.process(
+      in_file,
+      out_file,
+      { Subject: true, Date: true, From: true, To: true, CC: true }
+    )
+  end
+end

data/lib/mbox_header_scraper/mail.rb

@@ -0,0 +1,113 @@
+require 'nkf'
+
+# mail object
+class MboxHeaderScraper::Mail
+  @mail = nil
+
+  def initialize(mail_fp)
+    @mail = mail_fp
+  end
+
+  # rubocop:disable Metrics/AbcSize
+  def header_to_tsv(options)
+    # TODO: prevent undefined options
+
+    targets = get_target_keys(options)
+    result = initialize_result_hash(targets)
+
+    last_symbol = nil
+
+    IO.foreach(@mail) do |line|
+      line.chomp!
+
+      # this must be continuous value of previous header
+      if /^ / =~ line
+        # use extract_email_address because Subjects and Dates can't be multiple line
+        result[last_symbol].concat(extract_email_address(line.chomp)) if last_symbol
+
+        next
+      end
+
+      targets.each do |param|
+        matched = get_if_matched(param, line)
+
+        if matched
+          result[param].concat(matched) unless matched.empty?
+          last_symbol = param
+          break
+        end
+
+        last_symbol = nil
+      end
+
+      result = uniquify_result_hash(result)
+
+      # mail header must finish with the first empty line
+      return build_tsv(result, options) if /^$/ =~ line
+    end
+
+    # mail without empty line must be broken
+    raise 'this mail is broken'
+  end
+  # rubocop:enable Metrics/AbcSize
+
+  private
+
+  def get_target_keys(options)
+    options.select { |v| options[v] == true }.keys
+  end
+
+  def initialize_result_hash(targets)
+    result = {}
+    targets.each do |v|
+      result[v] = []
+    end
+
+    result
+  end
+
+  def uniquify_result_hash(hash)
+    hash.keys.each do |v|
+      hash[v].uniq!
+    end
+
+    hash
+  end
+
+  def get_if_matched(param, line)
+    return nil unless /^#{param}: / =~ line
+
+    if param == :Subject
+      subject = NKF.nkf('-mw', ($'))
+      [subject.chomp]
+    elsif param == :Date
+      [($').chomp]
+    else
+      extract_email_address(($').chomp)
+    end
+  end
+
+  def build_tsv(hash, options)
+    targets = get_target_keys(options)
+
+    tsv = ''
+
+    i = 0
+    loop do
+      line = []
+      targets.each do |v|
+        line << (hash[v][i] || '')
+      end
+
+      break if line.all? { |v| v == '' }
+      tsv += (line.join("\t") + "\n")
+      i += 1
+    end
+
+    tsv
+  end
+
+  def extract_email_address(str)
+    str.scan(/[0-9a-z.\+\-\_]+@[0-9a-z.\+\-\_]+/i)
+  end
+end

data/lib/mbox_header_scraper/scraper.rb

@@ -0,0 +1,60 @@
+# scrape mbox file
+class MboxHeaderScraper::Scraper
+  # rubocop:disable Metrics/AbcSize
+  def self.process(in_file, out_file, options = { Subject: true, Date: true, From: true, To: true, CC: true })
+    Tempfile.open('mbox_header_scraper_result') do |result_file|
+      result_file.write(insert_header(options))
+
+      tmp = nil
+
+      # to prevent nil error on first line
+      def tmp.closed?
+        true
+      end
+
+      IO.foreach(in_file) do |line|
+        if /^From / =~ line && !tmp.closed?
+          tmp.close(false)
+          result_file.write(single_mail_to_tsv(tmp, options))
+          tmp.delete
+        end
+
+        tmp = Tempfile.open('mbox_header_scraper_tmp') if tmp.closed?
+        tmp.write(line)
+      end
+
+      tmp.close(false)
+      result_file.write(single_mail_to_tsv(tmp, options))
+      tmp.delete
+
+      result_file.close(false)
+      FileUtils.copy_file(result_file.path, out_file)
+    end
+  end
+  # rubocop:enable Metrics/AbcSize
+
+  def self.check_in_file(filename)
+    return 'file does not exists.' unless File.exist?(filename)
+    nil
+  end
+
+  def self.check_out_file(filename)
+    return 'file already exists.' if File.exist?(filename)
+    nil
+  end
+
+  def self.single_mail_to_tsv(mail_file, options)
+    mail = MboxHeaderScraper::Mail.new(mail_file)
+    mail.header_to_tsv(options)
+  end
+
+  def self.insert_header(headers)
+    line = []
+
+    headers.select { |v| headers[v] == true }.keys.each do |v|
+      line << v
+    end
+
+    (line.join("\t") + "\n")
+  end
+end

data/lib/mbox_header_scraper/version.rb

@@ -0,0 +1,4 @@
+# About Version
+module MboxHeaderScraper
+  VERSION = '0.1.0'
+end

data/lib/mbox_header_scraper.rb

@@ -0,0 +1,10 @@
+require 'thor'
+
+require 'mbox_header_scraper/version'
+require 'mbox_header_scraper/cli'
+require 'mbox_header_scraper/scraper'
+require 'mbox_header_scraper/mail'
+
+# Main Class
+module MboxHeaderScraper
+end

data/mbox_header_scraper.gemspec

@@ -0,0 +1,43 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mbox_header_scraper/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'mbox_header_scraper'
+  spec.version       = MboxHeaderScraper::VERSION
+  spec.authors       = ['Yuya TAMANO']
+  spec.email         = ['everfree.main@gmail.com']
+
+  spec.summary       = 'tool to search mail header in mbox file'
+  spec.description   = 'See README.md for more information.'
+  spec.homepage      = 'https://github.com/tamano/mbox_header_scraper'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  if spec.respond_to?(:metadata)
+    # spec.metadata['allowed_push_host']
+  end
+
+  spec.add_dependency 'thor'
+  # spec.add_dependency 'mail'
+
+  spec.add_development_dependency 'bundler' # , '~> 1.9'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'rubocop'
+
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'awesome_print'
+  spec.add_development_dependency 'hirb'
+
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'guard-rubocop'
+  spec.add_development_dependency 'terminal-notifier-guard'
+end

metadata

@@ -0,0 +1,244 @@
+--- !ruby/object:Gem::Specification
+name: mbox_header_scraper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yuya TAMANO
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2015-05-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: codeclimate-test-reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hirb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: See README.md for more information.
+email:
+- everfree.main@gmail.com
+executables:
+- mbox_header_scraper
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .rubocop.yml
+- .ruby-version
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- circle.yml
+- exe/mbox_header_scraper
+- lib/mbox_header_scraper.rb
+- lib/mbox_header_scraper/cli.rb
+- lib/mbox_header_scraper/mail.rb
+- lib/mbox_header_scraper/scraper.rb
+- lib/mbox_header_scraper/version.rb
+- mbox_header_scraper.gemspec
+homepage: https://github.com/tamano/mbox_header_scraper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: tool to search mail header in mbox file
+test_files: []