km-export-processor 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 22bdc26538542d7771c79800ade24fe795a4a7a8
-  data.tar.gz: ea7099b19ff77405b98c0a4c6f96d743ce190176
+  metadata.gz: 0d3ac999d514d6a7934b57f20d9b0348cbf53511
+  data.tar.gz: ef204f4103e100e7be136c9ab076432e5c8652d2
 SHA512:
-  metadata.gz: 24daf8ac6bc1499676824342f3037353855f82a5a661d3669ac96f4837f3809ae29aa832efd786659d19d95665042f75cb5bafce6fea30c6fe6208cec2c68d0e
-  data.tar.gz: d32efbd85938438b6d19821e8e010a5d57685b2dbc1ba6ae44cf3192c6157cb01f4cac90982518171996b3df281fb47a5b9dfd7e1f10226ef6f4d082ddbc7403
+  metadata.gz: 480ffa22ca733b1fc31fdb447a729d3bb3c6431e171c66f8c9d680153d81b849a4cea05a58a4e4e7b119861b72a381f8f65724f8c1ff03dafba472bdcd4b132c
+  data.tar.gz: 5c2d7b3705625808e4a70fbc9f254494beb6e750814548aa1eb371005d6c1f2e348c3d4b51b18bbe4cabee179733ee7f2363a1f70be0678c4689f854bd3417df

data/bin/km-export-processor

@@ -26,8 +26,7 @@ class CLI < Thor
   def reimport(jsonfile, apikey)
     puts "Sending data from #{jsonfile} to KISSmetrics product with key: #{apikey}"
     reimporter = KMExport::Reimporter.new
-    file = File.open(jsonfile)
-    reimporter.send_to_KM(file.read, apikey)
+    reimporter.send_to_KM(jsonfile, apikey)
   end
 end
 

data/lib/alias_parser.rb

@@ -16,5 +16,7 @@ module KMExport
     new_file = File.open(jsonfile, "w+")
     new_file.write(JSON.pretty_generate(data))
     alias_file.write(JSON.pretty_generate(aliases))
+    new_file.close
+    alias_file.close
   end
 end

data/lib/converters/json_to_json.rb

@@ -14,5 +14,6 @@ module KMExport
     output = File.open(Time.now.to_i.to_s + "_result.json", "w+")
      
     output.write(JSON.pretty_generate(result))
+    output.close
   end
 end

data/lib/km-export-processor.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'csv'
 require 'kmts'
+require 'thread'
 
 module KMExport
 end

data/lib/reimporter.rb

@@ -1,16 +1,41 @@
+class JSONQueue < Queue
+  def initialize(opts)
+    super()
+    @file = opts[:file]
+    @counter = opts[:counter]
+  end
+
+  def add_to_queue
+    json = JSON.parse(File.read(@file))
+    json.each do |line|
+      self << line
+    end
+  end
+end
+
 module KMExport
   class Reimporter
     def send_to_KM(json, key)
-      KMTS.init(key,
-        :use_cron => false, 
-        :to_stderr => true)
-
-      logs = JSON.parse(json)
-      logs.each do |line|
-        send_line_to_KM(line)
-        STDOUT.print('.')
+      pool = 10
+      threads = []
+      queue = JSONQueue.new({file: json, counter: 0})
+
+      queue.add_to_queue
+
+      pool.times do
+        threads << Thread.new do
+          KMTS.init(key,
+          :use_cron => false, 
+          :to_stderr => true)
+          until queue.empty?
+            work_unit = queue.pop(true) rescue nil
+            send_line_to_KM(work_unit)
+          end
+        end
       end
 
+      threads.each { |t| t.join }
+
       STDOUT.print('Reimportation process completed.')
     end
 

data/spec/features/alias_parser_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe "alias_parser" do
+  before :all do
+    @aliasfilename = Time.now.to_i.to_s + "_test_alias.json"
+    testfile = File.open(@aliasfilename, "w+")
+    json = JSON.pretty_generate(
+      [
+        {_p2: "test", _p: "test1"},
+        {test2: "test2"},
+        {_p2: "test", _p: "test2"},
+        {test: "test"},
+        {_p2: "test", _p: "test3"},
+        {test3: "test3", test: "test3"}
+      ]
+      )
+    testfile.write(json)
+    testfile.close
+    KMExport.alias_parser(@aliasfilename)
+  end
+
+  it "removes aliases from JSON file" do
+    file = File.open(@aliasfilename)
+    json = JSON.parse(file.read)
+    json.select { |row| row["_p2"] }.length.should eq 0
+  end
+
+  it "creates new JSON file with aliases" do
+    file = File.open(Dir["*aliases.json"].first)
+    json = JSON.parse(file.read)
+    json.length.should eq 3
+  end
+
+  after :all do
+    File.delete(@aliasfilename)
+    File.delete(Dir["*aliases.json"].first)
+  end
+end

data/spec/features/json_to_csv_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe "json_to_csv" do
+  before :all do
+    @testfilename = Time.now.to_i.to_s + "_test.json"
+    testfile = File.open(@testfilename, "w+")
+    json = JSON.pretty_generate(
+      [
+        {test: "test"},
+        {test2: "test2"},
+        {test3: "test3", test: "test3"},
+        {test: "test"},
+        {test2: "test2"},
+        {test3: "test3", test: "test3"}
+      ]
+      )
+    testfile.write(json)
+    testfile.close
+    KMExport.json_to_csv(@testfilename)
+    @csv_files = Dir["*converter_result.csv"]
+  end
+
+  it "correctly converts JSON file to a CSV file" do
+    @csv_files.length == 1
+  end
+
+  it "uses the correct CSV format" do
+    csv = CSV.table(@csv_files.first)
+    csv.headers.length.should eq 3
+  end
+
+  after :all do
+    File.delete(@testfilename)
+    File.delete(@csv_files.first)
+  end
+end

data/spec/features/json_to_json_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe "json_to_json" do
+  before :all do
+    @testfilename1 = Time.now.to_i.to_s + "_test.json"
+    @testfilename2 = Time.now.to_i.to_s + "_test2.json"
+    @testfilename3 = Time.now.to_i.to_s + "_test3.json"
+    testfile = File.open(@testfilename1, "w+")
+    testfile2 = File.open(@testfilename2, "w+")
+    testfile3 = File.open(@testfilename3, "w+")
+    json = JSON.generate({test: "test"})
+    json2 = JSON.generate({test2: "test"})
+    json3 = JSON.generate({test3: "test"})
+    testfile.write(json)
+    testfile2.write(json2)
+    testfile3.write(json3)
+    testfile.close
+    testfile2.close
+    testfile3.close
+    KMExport.json_to_json
+    @json_files = Dir["*result.json"]
+  end
+
+  it "creates one JSON file from many" do
+    @json_files.length.should eq 1
+  end
+
+  it "creates a JSON file that contains the combined data" do
+    file = File.open(@json_files.first)
+    json = JSON.parse(file.read)
+    json.include?({"test" => "test"}) && json.include?({"test3" => "test"})
+  end
+
+  after :all do
+    File.delete(@testfilename1)
+    File.delete(@testfilename2)
+    File.delete(@testfilename3)
+    File.delete(@json_files.first)
+  end
+end

data/spec/features/reimporter_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe "reimporter" do
+  before do
+    @reimporter = KMExport::Reimporter.new
+    @json = JSON.pretty_generate(
+      [
+        {returning: "1", _p: "testuser", _t: 1384294380 },
+        {referrer: "Direct", _p: "testuser", _t: 1384275438, _n: "visited site" },
+        {_p2: "testuser2", _p: "testuser", _t: 1387799043 },
+        {returning: "1", _p: "testuser1", _t: 1384294380 },
+        {referrer: "Direct", _p: "testuser1", _t: 1384275438, _n: "visited site" },
+        {_p2: "testuser2", _p: "testuser1", _t: 1387799043 }
+      ]
+      )
+
+    @property_url = "http://trk.kissmetrics.com/s"
+    @event_url = "http://trk.kissmetrics.com/e"
+    @alias_url = "http://trk.kissmetrics.com/a"
+
+    WebMock.disable_net_connect!
+
+    stub_request(:get, @property_url).with(:query => hash_including({"returning" => "1"}))
+    stub_request(:get, @event_url).with(:query => hash_including({"_n" => "visited site"}))
+    stub_request(:get, @alias_url).with(:query => hash_including({"_n" => "testuser2"}))
+
+    @reimporter.send_to_KM(@json, "1234")
+  end
+
+  it "succesfully sends a request for an event" do
+    a_request(:get, @event_url).with(:query => hash_including({"_n" => "visited site"})).should have_been_made.times(2)
+  end
+
+  it "succesfully sends a request for a property" do
+    a_request(:get, @property_url).with(:query => hash_including({"returning" => "1"})).should have_been_made.times(2)
+  end
+
+  it "succesfully sends a request for an alias" do
+    a_request(:get, @alias_url).with(:query => hash_including({"_n" => "testuser2"})).should have_been_made.times(2)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'rspec'
+require 'km-export-processor'
+require 'webmock/rspec'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: km-export-processor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Clay Whitley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-07 00:00:00.000000000 Z
+date: 2014-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kmts
@@ -80,6 +80,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '10.1'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
@@ -121,6 +135,11 @@ files:
 - lib/converters/json_to_json.rb
 - lib/km-export-processor.rb
 - lib/reimporter.rb
+- spec/features/alias_parser_spec.rb
+- spec/features/json_to_csv_spec.rb
+- spec/features/json_to_json_spec.rb
+- spec/features/reimporter_spec.rb
+- spec/spec_helper.rb
 - bin/km-export-processor
 homepage: https://github.com/kissmetrics/km-export-processor
 licenses: []
@@ -145,5 +164,10 @@ rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: A gem for processing KM exports
-test_files: []
+test_files:
+- spec/features/alias_parser_spec.rb
+- spec/features/json_to_csv_spec.rb
+- spec/features/json_to_json_spec.rb
+- spec/features/reimporter_spec.rb
+- spec/spec_helper.rb
 has_rdoc: