pinport 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4e6decec98406cbf6572de83a023f6d746965bfb
+  data.tar.gz: 61d9b68ac81489cdde13e3fed979daa1f6457e37
+SHA512:
+  metadata.gz: e28234ce6aacc3642a7dc037cc71c7c61723c6a572dee7325c00a1fef2e182f60d7b999c3dc99874be702b4c3e72c1d2fd3a78b978ef9ceede92dd3bf5f9f16a
+  data.tar.gz: 7770d071f68f284580dd1fe4ff9729fdafd3e1c70aba9829f323672859540b53c590f802f5a28c8399e4e304530538017a4c6e84928ed757671c210ace6348b5

data/LICENSE.md

@@ -0,0 +1,21 @@
+###The MIT License (MIT)
+
+Copyright (c) 2013 Francis San Juan http://github.com/ftsanjuan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# pinport
+
+Pinport is a PIN importing command line utility written in Ruby.
+
+Currently supports importing to a MySQL database.
+
+## Installation
+
+``` sh
+gem install pinport
+```
+
+
+## Configuration
+
+A `config.yml` file specifying details for your MySQL database is required.  Pinport uses the file 
+
+### Generating a config.yml file
+
+Use `pinport generate` to generate the required `config.yml` file in the current directory.
+
+Pinport uses the `mysql2` gem to establish the connection to the database. For available database connection options, refer to: https://github.com/brianmario/mysql2#connection-options.
+
+
+## Usage
+
+### Importing a single .txt file:
+
+``` sh
+pinport import FILE
+```
+
+FILE should contain one line per item to be imported.
+
+### Importing a folder of .txt files:
+``` sh
+pinport import FOLDER
+```
+
+
+## Development
+Clone this repository using `git clone`.
+
+Navigate to the directory of cloned repository and run `rake` to compile and install the gem from 
+source.
+
+To do the tasks separately:
+
+- `rake build` to compile the gem
+- `rake install` to install the gem.

data/bin/pinport

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'pinport'
+
+class Cli < Thor
+  include Thor::Actions
+  package_name "Pinport"
+
+  # Imports a file contaning pins into the database
+  desc "import FILE", "Imports pins from FILE"
+  method_option :table, :default => nil, :aliases =>  '-t'
+  method_option :column, :default => nil, :aliases => '-c'
+  method_option :filter, :default => nil, :aliases => '-f'
+  method_option :fix_newlines, :type => :boolean, :default => true, :aliases => '-newlines'
+  method_option :config_file, :default => 'config.yml', :aliases => '-config'
+  def import(file)
+    table = options[:table]
+    column = options[:column]
+    filter = options[:filter]
+    fix_newlines = options[:fix_newlines]
+    config_file = options[:config_file]
+
+    begin
+      test = File.open(file)
+    rescue => e
+      puts e.message
+      exit 1
+    end
+
+    begin
+      check_config config_file
+    rescue => e
+      puts e.message
+    else
+      # Otherwise... attempt to import pins from FILE into database
+      puts "Attempting import using following settings:"
+      puts "PINs source: #{file}"
+      puts "Target Database: #{Pinport.database_name}"
+      puts "Table: #{Pinport.table_name}"
+      puts "Column: #{Pinport.column_name}"
+      if yes? "Continue with import?"
+        Pinport.import_file(file, table, column, filter, fix_newlines)
+        say "pinport import complete.", :green
+      else
+        say "Import cancelled.", :red
+        exit 1
+      end
+    end
+  end
+
+  # Imports a folder of files containing pins into the database
+  desc "import_folder FOLDER", "Imports pins from files in a FOLDER"
+  method_option :extension, :default => 'txt', :aliases => '-ext'
+  method_option :table, :default => nil, :aliases =>  '-t'
+  method_option :column, :default => nil, :aliases => '-c'
+  method_option :filter, :default => nil, :aliases => '-f'
+  method_option :fix_newlines, :type => :boolean, :default => true, :aliases => '-newlines'
+  method_option :config_file, :default => 'config.yml', :aliases => '-config'
+  def import_folder(folder)
+    extension = options[:extension]
+    table = options[:table]
+    column = options[:column]
+    filter = options[:filter]
+    fix_newlines = options[:fix_newlines]
+    config_file = options[:config_file]
+
+    begin
+      check_config config_file
+    rescue => e
+      puts e.message
+    else
+      # Otherwise... attempt to import pins from FILE into database
+      puts "Attempting import using following settings:"
+      file_targets = Dir.glob("#{folder}/*.#{extension}")
+      puts "Found: #{file_targets.count} .#{extension} files in #{folder}:"
+      file_targets.each do |f|
+        puts "  #{f}"
+      end
+      puts "Target:"
+      puts "  Database: #{Pinport.database_name}"
+      puts "  Table: #{Pinport.table_name}"
+      puts "  Column: #{Pinport.column_name}"
+
+      if yes? "Continue with import?"
+        Pinport.import_folder(folder, extension, table, column, filter, fix_newlines)
+        say "pinport import_folder complete.", :green
+      else
+        say "Import cancelled.", :red
+        exit 1
+      end
+    end
+  end
+
+  # Generates a base config file
+  desc "generate", "Generates a base config.yml file in the current directory"
+  method_option :output, :default => 'config.yml', :aliases => '-o'
+  def generate(output='config.yml')
+    output = "#{Dir.pwd}/#{options[:output]}"
+    say "Generating a config file in: #{output}"
+    Pinport::Generator.generate_config(output)
+    say "pinport generate complete.", :green
+  end
+
+  # Verifies if the supplied config file is valid
+  desc "check_config CONFIG_FILE", "Checks pinport's configuration and initializes it"
+  def check_config(config_file)
+    # Prompts user to generate a default config.yml file if it doesn't exist
+    if !File.exists?("#{Dir.pwd}/config.yml")
+      if yes? "No config file found in current directory, generate one?"
+        # invoke generate with default parameters
+        invoke :generate, ["config.yml"]
+      else
+        raise "Config file not found."
+      end
+    end
+
+    # Prompts user to use the default config file
+    if config_file.empty?
+      if yes? "Config file not specified, use default config.yml?"
+        config_file = 'config.yml'
+      else
+        raise "No config file specified used."
+      end
+    end
+
+    # Convert config file path to be absolute
+    config_file = File.absolute_path(config_file)
+
+    # Attempt to initialize the module
+    begin
+      # Check if module can successfully use the supplied config_file
+      Pinport.initialize(config_file)
+    rescue => e
+      puts e.message
+      say "Unable to use #{config_file} to configure Pinport.", :red
+      say "Verify that it is a valid YAML file, or try running 'pinport generate' to create a new config file.", :red
+    end
+  end
+end
+
+Cli.start(ARGV)

data/lib/pinport/core_ext/hash.rb

@@ -0,0 +1,15 @@
+class Hash
+  # Returns a hash containing symbolized keys
+  #
+  # @param hash [Hash]
+  # @return [Hash] The same hash with symbolized keys
+  def symbolize_keys
+    return nil unless hash
+    result = {}
+    each_key do |key|
+      result[key.to_sym] = self[key]
+    end
+    result
+  end
+
+end

data/lib/pinport/error.rb

@@ -0,0 +1,30 @@
+module Pinport
+  class Error < StandardError
+  end
+
+  class DatabaseConnectionError < Error
+    def message
+      @msg = "An error occurred while establishing database connection.\n"
+      @msg += "Please verify your database connection information."
+      @msg
+    end
+  end
+
+  class InvalidFileError < SystemCallError
+    def message
+      "An invalid file or directory was specified. Please try again."
+    end
+  end
+
+  class UnexpectedArgumentError < ArgumentError
+    def message
+      "Encountered an unexpected argument."
+    end
+  end
+
+  class DirectoryArgumentError < UnexpectedArgumentError
+    def message
+      "A directory was supplied as an argument to a method expecting a file argument."
+    end
+  end
+end

data/lib/pinport/generators/generator.rb

@@ -0,0 +1,11 @@
+module Pinport
+  module Generator
+    def self.templates_path
+      File.expand_path('../templates', __FILE__)
+    end
+
+    def self.generate_config(output = 'config.yml')
+      FileUtils.copy("#{templates_path}/config/config.yml", "#{output}")
+    end
+  end
+end

data/lib/pinport/generators/templates/config/config.yml

@@ -0,0 +1,41 @@
+# Settings used to connect to the database
+#
+# refer to mysql2 gem documentation for additional connection options
+#   https://github.com/brianmario/mysql2#connection-options
+#
+database:
+  host: localhost
+  username:
+  password:
+  database:
+  socket:
+
+# Database schema settings
+#
+# table:
+#   the table name to import pins into
+# column:
+#   the name of the column (field) to insert pins into
+schema:
+  table: pins
+  column: pin
+
+# Terminal output (notification) settings
+#
+# import:
+#   every:
+#     prints a confirmation message every nth time a pin is imported
+#     reduce number to increase import performance
+#
+notifications:
+  import:
+    every: 20000
+
+# Various debugging settings
+#
+# errors:
+#   verbose:
+#     if set to true, will display error backtraces
+debug:
+  errors:
+    verbose: false

data/lib/pinport/parser.rb

@@ -0,0 +1,74 @@
+module Pinport
+  class Parser
+    require 'tempfile'
+    require 'shellwords'
+
+    class << self
+      # Fixes linebreaks on files and returns result
+      # @param file [String] path to the file to be processed by dos2unix
+      # @param output_path [String] path to file where output should be returned, default: same as file
+      # @return [String] path to file that was processed
+      def dos2unix(file, output_path = nil)
+        puts "Fixing newlines for #{file} ..."
+        if File.exists?(file)
+          input = File.open(file)
+          output = Tempfile.new("/tmp")
+          regex = /\r\n/
+          newstring = "\n"
+          while (line = input.gets) do
+            output_line = line.gsub(regex, newstring)
+            output << output_line
+          end
+          input.close
+          output.close
+          output_path = file unless output_path != nil
+          system("mv #{Shellwords.shellescape(output.path)} #{Shellwords.shellescape(output_path)}")
+          puts "Parser: Corrected newlines."
+          return output_path
+          # return "Newlines were successfully fixed for #{file}"
+        else
+          puts "Specified file does not exist."
+          return nil
+        end
+      end
+
+      # Strips characters to be filtered and returns result
+      # @param file [String] path to the file to be processed
+      # @param chars [String] a string of characters to be filtered out
+      # @param output_path [String] path to file where output should be returned, defaults to same path as `file` parameter
+      # @return [String] path to file that was processed
+      def strip_chars(file, chars, output_path = nil)
+        puts "Filtering: #{file} ..."
+        if File.exists?(file)
+          input = File.open(file)
+          output = Tempfile.new("/tmp")
+          while (line = input.gets) do
+            output_line = line.gsub!(chars, '')
+            output << output_line
+          end
+          input.close
+          output.close
+          output_path = file unless output_path != nil
+          system("mv #{Shellwords.shellescape(output.path)} #{Shellwords.shellescape(output_path)}")
+          puts "Parser: Filtered #{chars}."
+          return output_path
+        else
+          puts "Specified file does not exist."
+          return nil
+        end
+      end
+
+      # Runs `sort` command to sort contents of a text file. Returns true if successfully outputted to a file; otherwise prints the contents of the result
+      # @param input [String] path to the file to be sorted
+      # @param output_path [String] the path to
+      def sort_file_contents(input, output_path = nil)
+        puts "Sorting: #{input} ..."
+        # Shellescape arguments to prevent errors
+        system("sort #{Shellwords.shellescape(input)} -o #{Shellwords.shellescape(output_path)}") unless output_path == nil
+        return true
+        # otherwise: print the result
+        `sort #{input}`
+      end
+    end
+  end
+end

data/lib/pinport/version.rb

@@ -0,0 +1,3 @@
+module Pinport
+  VERSION = "0.0.1"
+end

data/lib/pinport.rb

@@ -0,0 +1,177 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'mysql2'
+require 'yaml'
+require 'sequel'
+require 'fileutils'
+require 'pinport/parser'
+require 'pinport/error'
+require 'pinport/core_ext/hash'
+require 'pinport/generators/generator'
+
+module Pinport
+  @db = nil
+  @settings = nil
+  @settings_file = nil
+  @notifications = nil
+  @verbose_errors = nil
+
+  def self.initialize(file)
+    @db = db_connection(self.settings['database'])
+    @verbose_errors = self.settings['debug']['errors']['verbose']
+    self.settings_file = file
+  end
+
+  def self.db
+    @db ||= db_connection(self.settings['database'])
+  end
+
+  def self.settings
+    @settings ||= YAML.load_file(self.settings_file)
+  end
+
+  def self.settings=(new_settings)
+    @settings = new_settings
+  end
+
+  def self.schema
+    @settings['schema'] ||= self.settings['schema']
+  end
+
+  def self.database_name
+    @settings['database']['database'] ||= self.settings['database']['database']
+  end
+
+  def self.table_name
+    @settings['schema']['table'] ||= self.settings['schema']['table']
+  end
+
+  def self.column_name
+    @settings['schema']['column'] ||= self.settings['schema']['column']
+  end
+
+  # Loads a config file to use with pinport
+  # @param file [String] the config file to load
+  # @return [Hash] Parsed settings as a hash
+  def self.load_config_file(file)
+    if File.exists?(file)
+      puts "Loading config file: #{file}"
+      return YAML.load_file(file)
+    else
+      puts "Could not find config file: #{file}"
+    end
+  end
+
+  # Updates settings_file and settings vars if specified
+  # config file can be found
+  # @param file [String] the config file to be loaded for settings
+  def self.settings_file=(file)
+    if File.exists?(file)
+      @settings_file = file
+      self.settings = load_config_file(@settings_file)
+      return true
+    else
+      raise "Specified settings file could assigned."
+    end
+  end
+
+  def self.settings_file
+    @settings_file ||= 'config.yml'
+  end
+
+  def self.root
+    File.expand_path '../..', __FILE__
+  end
+
+  def self.notifications
+    @notifications ||= self.settings['notifications']
+  end
+
+  # Establishes a connection to the database
+  # @param db_config [String] path to a YAML file specifying database connection details, refer to mysql2 connection options for available settings (https://github.com/brianmario/mysql2#connection-options).
+  # @return [Sequel::Mysql2::Database]
+  def self.db_connection(db_config)
+    db_config = db_config.symbolize_keys
+    db_config[:test] = true
+    begin
+      Sequel.mysql2(db_config)
+    rescue => e
+      puts e.message
+      if @verbose_errors
+        puts "Trace:\n"
+        puts e.backtrace
+      end
+      exit 1
+    end
+  end
+
+  # Imports pins into database
+  #
+  # @param file [String] path to the file to be imported
+  # @param table [String] the table name that pins should be imported into
+  # @param column [String] the column name pins should be inserted into, default: "pin"
+  # @param filter [String] a string of characters to filter out from each pin
+  # @param fix_newlines [Boolean] specifies whether to fix newlines (in case of cross-OS incompatibilities), default: true
+  def self.import_file(file, table = nil, column = nil, filter = nil, fix_newlines = true)
+    # convert supplied file path to absolute
+    file_original = "#{file}"
+    file = File.absolute_path("#{file}")
+    if !File.exists?(file)
+      raise Pinport::InvalidFileError
+    elsif File.directory?(file)
+      raise Pinport::DirectoryArgumentError
+    end
+
+    # escape filename as precaution
+    # file = Shellwords.shellescape(file)
+    source = File.open(file)
+    source_dirname = File.dirname(file)
+    source_basename = File.basename(file, '.txt')
+    cleaned_filename = "#{source_dirname}/#{source_basename}-cleaned.txt"
+    sorted_filename = "#{source_dirname}/#{source_basename}-sorted.txt"
+
+    # clean up txt files
+    cleaned = Pinport::Parser.dos2unix(file) unless fix_newlines == false
+    cleaned = Pinport::Parser.strip_chars(cleaned, filter, cleaned_filename) unless filter == nil
+
+    # perform the sort, if successful...
+    if Pinport::Parser.sort_file_contents(cleaned, sorted_filename)
+
+      # build a Sequel dataset and insert pins into it
+      # retrieve table/column names from schema if not specified
+      puts  "Importing..."
+      table = self.table_name if table == nil
+      column = self.column_name if column == nil
+      pins = self.db[table.to_sym]
+
+      # open the sorted file and insert each pin line-by-line
+      count = 0
+      start_time = Time.now
+      File.open(sorted_filename).each_line do |line|
+        pins.insert(column.to_sym => line.chomp)
+        count += 1
+        puts "Imported #{count} pins." if (count % self.notifications['import']['every'] == 0)
+      end
+      puts "Finished importing #{count} pins into #{table}."
+
+      # remove any temporary files that were created
+      File.delete(cleaned_filename) if File.exists?(cleaned_filename)
+      File.delete(sorted_filename) if File.exists?(sorted_filename)
+    else
+      puts "Error encountered while sorting pins in file."
+      return false
+    end
+  end
+
+  # Imports pins from folder containing pin files
+  #
+  # Accepts the same parameters as `import_file` along with the following:
+  # @param extension [String] file extension of files to be imported, default: 'txt'
+  def self.import_folder(folder, extension = "txt", table = nil, column = nil, filter = nil, fix_newlines = true)
+    folder = File.absolute_path(folder)
+    Dir.glob("#{folder}/*.#{extension}").each do |file|
+      file = File.absolute_path("#{file}")
+      import_file(file, table, column, filter, fix_newlines)
+    end
+  end
+end

data/pinport.gemspec

@@ -0,0 +1,27 @@
+require File.expand_path('lib/pinport/version')
+
+Gem::Specification.new do |s|
+  s.name          = 'pinport'
+  s.version       = Pinport::VERSION
+  s.date          = '2013-04-29'
+  s.summary       = %q{A PIN importing tool}
+  s.description   = %q{Pinport is a command-line tool for importing a large list of PINs (Personal Identification Numbers) into a MySQL Database}
+  s.authors       = ['Francis San Juan']
+  s.email         = 'francis.sanjuan@gmail.com'
+  s.license       = 'MIT'
+  s.licenses      = ['MIT']
+  s.homepage      = 'https://github.com/ftsanjuan/pinport'
+  s.require_paths = ['lib']
+  s.files         =  %w(README.md LICENSE.md README.md pinport.gemspec)
+  s.files        +=  Dir.glob("lib/**/*.rb")
+  s.files        +=  Dir.glob("lib/**/generators/templates/**/*")
+  s.executables  << 'pinport'
+
+  s.add_runtime_dependency  'bundler', '~> 1.0'
+  s.add_runtime_dependency 'mysql2', '~> 0.3.11'
+  s.add_runtime_dependency 'sequel', '~> 3.46.0'
+
+  s.add_development_dependency 'rake', '~> 10.0.4'
+  s.add_development_dependency 'yard', '~> 0.8.6.1'
+  s.add_development_dependency 'thor', '~> 0.18.1'
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: pinport
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Francis San Juan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.11
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.46.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.46.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.0.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 10.0.4
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.6.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.6.1
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.18.1
+description: Pinport is a command-line tool for importing a large list of PINs (Personal
+  Identification Numbers) into a MySQL Database
+email: francis.sanjuan@gmail.com
+executables:
+- pinport
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- LICENSE.md
+- pinport.gemspec
+- lib/pinport/core_ext/hash.rb
+- lib/pinport/error.rb
+- lib/pinport/generators/generator.rb
+- lib/pinport/parser.rb
+- lib/pinport/version.rb
+- lib/pinport.rb
+- lib/pinport/generators/templates/config/config.yml
+- bin/pinport
+homepage: https://github.com/ftsanjuan/pinport
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: A PIN importing tool
+test_files: []
+has_rdoc: