km-export-processor 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d3ac999d514d6a7934b57f20d9b0348cbf53511
-  data.tar.gz: ef204f4103e100e7be136c9ab076432e5c8652d2
+  metadata.gz: a125a73e6d91278dc3758a499d66e43ffa32339c
+  data.tar.gz: 252a99be14e0aab383afc5b24fcb73c6f1357d4f
 SHA512:
-  metadata.gz: 480ffa22ca733b1fc31fdb447a729d3bb3c6431e171c66f8c9d680153d81b849a4cea05a58a4e4e7b119861b72a381f8f65724f8c1ff03dafba472bdcd4b132c
-  data.tar.gz: 5c2d7b3705625808e4a70fbc9f254494beb6e750814548aa1eb371005d6c1f2e348c3d4b51b18bbe4cabee179733ee7f2363a1f70be0678c4689f854bd3417df
+  metadata.gz: c4ba69d23ad4e905ba14b3ee7e13be375df5a3eb2b8ae38900dcafb6a43ba1bf3d8c2f90310bf4a333045fac385c7d2a198cdedec40f0c55d6124cd191e27730
+  data.tar.gz: 4cea71894f8a8b9a362656a374d2df5dbfe840c7ced8af3034c195236d2f4b7773c0ea82c1f145f86e69af29ebca2492cf6e12e4e3acf193213086ac1adaf8b2

data/bin/km-export-processor

@@ -4,25 +4,31 @@ require 'thor'
 require 'km-export-processor'
 
 class CLI < Thor
-  desc "json_to_json", "Processes all JSON in current directory, to standard format."
-  def json_to_json
-    KMExport.json_to_json
-    puts "Processing JSON files in this directory, and outputting new combined file"
+  desc "json_to_json [JSONFILE]", "Processes a non-standard KM JSON file, into a standard format"
+  def json_to_json(jsonfile)
+    puts "Processing nonstandard JSON file into standard format"
+    KMExport.json_to_json(jsonfile)
   end
 
-  desc "json_to_csv [JSONFILE]", "Takes a standard JSON file as input, and converts it to CSV"
+  desc "json_compiler", "Compiles all KISSmetrics JSON files located in current directory, into a single (nonstandard) file"
+  def json_compiler
+    KMExport.json_compiler
+  end
+
+
+  desc "json_to_csv [JSONFILE]", "Takes a KISSmetrics JSON file as input, and converts it to CSV"
   def json_to_csv(jsonfile)
-    KMExport.json_to_csv(jsonfile)
     puts "Converting file #{jsonfile} to CSV"
+    KMExport.json_to_csv(jsonfile)
   end
 
-  desc "parse_aliases [JSONFILE]", "Takes JSON as input, and separates the alias calls into their own JSON file"
+  desc "parse_aliases [JSONFILE]", "Takes KISSmetrics JSON as input, and separates the alias calls into their own JSON file"
   def parse_aliases(jsonfile)
-    KMExport.alias_parser(jsonfile)
     puts "Parsing aliases out of #{jsonfile} and into their own JSON file"
+    KMExport.alias_parser(jsonfile)
   end
 
-  desc "reimport [JSONFILE] [APIKEY]", "Takes JSON, and a KM API key as input, and sends the data to KISSmetrics"
+  desc "reimport [JSONFILE] [APIKEY]", "Takes KISSmetrics JSON, and a KM API key as input, and sends the data to KISSmetrics"
   def reimport(jsonfile, apikey)
     puts "Sending data from #{jsonfile} to KISSmetrics product with key: #{apikey}"
     reimporter = KMExport::Reimporter.new

data/lib/alias_parser.rb

@@ -1,22 +1,28 @@
 module KMExport
   def self.alias_parser(jsonfile)
-    file = File.open(jsonfile)
-    alias_file = File.open(Time.now.to_i.to_s + "_aliases.json", "w+")
-    json = JSON.parse(file.read)
-    aliases = []
-    data = []
+    input = IO.open(IO.sysopen(jsonfile))
+    alias_filename = Time.now.to_i.to_s + "_aliases.json"
+    new_filename = Time.now.to_i.to_s + "_result.json"
+    File.open(alias_filename, "w+").close
+    File.open(new_filename, "w+").close
+    alias_output = IO.open(IO.sysopen(alias_filename, "w"), "w")
+    data_output = IO.open(IO.sysopen(new_filename, "w"), "w")
 
-    json.each do |row|
-      if row["_p2"]
-        aliases << row
+    until input.eof?
+      line = JSON.parse(input.readline)
+      if line["_p2"]
+        alias_output.write(JSON.generate(line))
+        alias_output.write("\n")
       else
-        data << row
+        data_output.write(JSON.generate(line))
+        data_output.write("\n")
       end
     end
-    new_file = File.open(jsonfile, "w+")
-    new_file.write(JSON.pretty_generate(data))
-    alias_file.write(JSON.pretty_generate(aliases))
-    new_file.close
-    alias_file.close
+
+    input.close
+    alias_output.close
+    data_output.close
+
+    File.delete(jsonfile)
   end
 end

data/lib/converters/json_compiler.rb

@@ -0,0 +1,16 @@
+module KMExport
+  def self.json_compiler
+    files = Dir["*.json"]
+    result = []
+    filename = "#{Time.now.to_i.to_s}_result.json"
+    File.open(filename, "w+").close
+    output = IO.open(IO.sysopen(filename, "w"), "w")
+
+    files.each do |file|
+      input = IO.open(IO.sysopen(file))
+      until input.eof?
+        output.write(input.readline)
+      end
+    end
+  end
+end

data/lib/converters/json_to_csv.rb

@@ -1,27 +1,24 @@
 module KMExport
   def self.json_to_csv(jsonfile)
-    default_headers = ["_n", "_p", "_t", "_p2", "_a"]
     headers = []
-    file = File.open(jsonfile)
-    json = JSON.parse(file.read)
-    results = []
-    json.each do |row|
-      headers = headers | row.keys
+    csv_filename = Time.now.to_i.to_s + "_converter_result.csv"
+    input = IO.open(IO.sysopen(jsonfile))
+    output = CSV.open(csv_filename, "w+")
+    
+    until input.eof?
+      headers = headers | JSON.parse(input.readline).keys
     end
 
-    json.each do |row|
+    output << headers
+    input.rewind
+
+    until input.eof?
+      row_data = JSON.parse(input.readline)
       row_result = {}
       headers.each do |header|
-        row_result[header.to_sym] = row[header] || ""
-      end
-      results << row_result
-    end
-    
-    CSV.open(Time.now.to_i.to_s + "_converter_result.csv", "wb") do |csv|
-      csv << results.first.keys
-      results.each do |hash|
-        csv << hash.values
+        row_result[header.to_sym] = row_data[header] || ""
       end
+      output << row_result.values
     end
   end
 end

data/lib/converters/json_to_json.rb

@@ -1,17 +1,13 @@
 module KMExport
-  def self.json_to_json
-    files = Dir["*.json"]
- 
+  def self.json_to_json(filename)
+    input = IO.open(IO.sysopen(filename))
     result = []
-     
-    files.each do |file|
-      json = File.open(file, "r")
-      json.readlines.each do |line|
-        result << JSON.parse(line.chomp)
-      end
+
+    until input.eof?
+      result << JSON.parse(input.readline)
     end
      
-    output = File.open(Time.now.to_i.to_s + "_result.json", "w+")
+    output = File.open(Time.now.to_i.to_s + "_STANDARD.json", "w+")
      
     output.write(JSON.pretty_generate(result))
     output.close

data/lib/km-export-processor.rb

@@ -8,5 +8,6 @@ end
 
 require_relative 'converters/json_to_json'
 require_relative 'converters/json_to_csv'
+require_relative 'converters/json_compiler'
 require_relative 'reimporter'
 require_relative 'alias_parser'

data/lib/reimporter.rb

@@ -1,15 +1,16 @@
 class JSONQueue < Queue
   def initialize(opts)
     super()
-    @file = opts[:file]
+    @io = IO.open(IO.sysopen(opts[:file]))
     @counter = opts[:counter]
   end
 
   def add_to_queue
-    json = JSON.parse(File.read(@file))
-    json.each do |line|
-      self << line
+    100.times do
+      self << JSON.parse(@io.readline)
     end
+    @counter += 100
+    puts "Adding 100 lines to queue, #{counter} so far"
   end
 end
 
@@ -28,6 +29,7 @@ module KMExport
           :use_cron => false, 
           :to_stderr => true)
           until queue.empty?
+            queue.add_to_queue if queue.length < 50
             work_unit = queue.pop(true) rescue nil
             send_line_to_KM(work_unit)
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: km-export-processor
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - Clay Whitley
@@ -131,6 +131,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/alias_parser.rb
+- lib/converters/json_compiler.rb
 - lib/converters/json_to_csv.rb
 - lib/converters/json_to_json.rb
 - lib/km-export-processor.rb