piplcollector 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3586ce00624a23fabe60ec6135415221723b8ec0
+  data.tar.gz: 41dfc1accf74bc77c0c8a09ad0374f0b622d8592
+SHA512:
+  metadata.gz: 43229a308129b7211ffeb4401fff72ed2a6d46479c300620431de696a675f9679d00975ad4eb506d765593c018fa4b859f82249174baac0ea884e530713363c1
+  data.tar.gz: fcc01c2ad1ec8c8029b9d8cc605de476f481b2784e5adffb47839e3a64ddbaee915a1ff4a477303498d242209f2a5ab85983f3ee8af3f64b0ea1826580b7ee27

data/lib/piplcollector.rb

@@ -0,0 +1,116 @@
+require 'piplrequest'
+require 'json'
+require 'pry'
+
+class PiplCollector
+  def initialize(input_dir, output_dir, output_append_dir, id_field, ignore_files, api_key, field_mapping)
+    @input_dir = input_dir
+    @output_dir = output_dir
+    @output_append_dir = output_append_dir
+    @id_field = id_field
+    @ignore_files = ignore_files
+    @api_key = api_key
+    @field_mapping = field_mapping
+    @already_collected = load_output_files
+  end
+
+  # Load the output files into already_collected 
+  def load_output_files
+    collected = []
+
+    # Make a list of all saved files
+    Dir.foreach(@output_dir) do |file|
+      next if file == '.' or file == '..'
+      collected.push(file.gsub(".json", ""))
+    end
+    
+    return collected
+  end
+
+  # Save output file
+  def save_output_file(output_item, data_item)
+    id = gen_filename_from_id(data_item)
+    File.write(@output_dir+"/"+id+".json", output_item)
+    @already_collected.push(id)
+  end
+
+  # Generates a file-safe name from the id field
+  def gen_filename_from_id(data_item)
+    data_item[@id_field].gsub(":", "").gsub("/", "").gsub(".", "")
+  end
+
+  # Checks if it is already collected
+  def was_collected?(data_item)
+    @already_collected.include?(gen_filename_from_id(data_item))
+  end
+
+  # Get info on person from pipl
+  def get_person(data_item)
+    sleep(1)
+    
+    # Get data from Pipl
+    p = PiplRequest.new(@api_key, @field_mapping)
+    output = p.get_data(data_item)
+
+    # Handle output
+    save_output_file(output, data_item) if output
+    return JSON.parse(output) if output
+  end
+
+  # Gets content for already collected person
+  def get_already_collected_person(data_item)
+    filename = @output_dir+"/"+gen_filename_from_id(data_item)+".json"
+    return file = JSON.parse(File.read(filename))
+  end
+
+  # Process file
+  def process(file)
+    data = JSON.parse(File.read(file))
+    outfile = Array.new
+
+    # Go through each item in file
+    data.each do |item|
+      if !was_collected?(item)
+        item[:pipl] = get_person(item)
+      else
+        item[:pipl] = get_already_collected_person(item)
+      end
+      outfile.push(item)
+    end
+
+    JSON.pretty_generate(outfile)
+  end
+
+  # Create if they don't exist
+  def create_write_dirs(dir)
+    dirs = dir.split("/")
+    dirs.delete("")
+    overallpath = ""
+    dirs.each do |d|
+      Dir.mkdir(overallpath+"/"+d) if !File.directory?(overallpath+"/"+d)
+      overallpath += ("/"+d)
+    end
+  end
+
+  # Figure out where to write it
+  def get_write_dir(dir, file)
+    dir_save = dir.gsub(@input_dir, @output_append_dir)
+    return dir_save+"/"+file
+  end
+
+  # Run on files
+  def run(dir)
+    Dir.foreach(dir) do |file|
+      next if file == '.' or file == '..'
+      if File.directory?(dir+"/"+file)
+        run(dir+"/"+file)
+      elsif file.include?(".json") && !file.include?(@ignore_files)
+        if !File.exist?(get_write_dir(dir, file))
+          with_pipl = process(dir+"/"+file)
+          create_write_dirs(dir.gsub(@input_dir, @output_append_dir))
+          File.write(get_write_dir(dir, file), with_pipl)
+        end
+      end
+    end 
+  end
+end

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: piplcollector
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- M. C. McGrath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-21 00:00:00.000000000 Z
+dependencies: []
+description: Gets data from Pipl for dir of files
+email: shidash@shidash.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/piplcollector.rb
+homepage: https://github.com/TransparencyToolkit/piplcollector
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Gets data from Pipl for dir of files
+test_files: []
+has_rdoc: