apple_epf 1.0.0

data/lib/apple_epf/parser.rb

@@ -0,0 +1,95 @@
+module AppleEpf
+  class Parser
+    FIELD_SEPARATOR = 1.chr
+    RECORD_SEPARATOR = 2.chr + "\n"
+    COMMENT_CHAR = '#'
+
+    attr_accessor :filename, :header_info, :footer_info
+
+    def initialize(filename)
+      @filename = filename
+      @header_info = {}
+      @footer_info = {}
+    end
+
+    def parse_metadata
+      begin
+        parse_file
+        load_header_info
+        load_footer_info
+        @header_info.merge(@footer_info)
+      ensure
+        close_file
+      end
+    end
+
+    def process_rows(&block)
+      File.foreach( @filename, RECORD_SEPARATOR ) do |line|
+        unless line[0].chr == COMMENT_CHAR
+          line = line.chomp( RECORD_SEPARATOR )
+          block.call( line.split( FIELD_SEPARATOR, -1) ) if block_given?
+        end
+      end
+    end
+
+    private
+
+    def parse_file
+      @file = File.new( @filename, 'r', encoding: 'UTF-8' )
+    end
+
+    def close_file
+      @file.close if @file
+    end
+
+    def read_line(accept_comment = false)
+      valid_line = false
+      until valid_line
+        begin
+          line = @file.readline( RECORD_SEPARATOR )
+        rescue EOFError => e
+          return nil
+        end
+        valid_line = accept_comment ? true : !line.start_with?( COMMENT_CHAR )
+      end
+      line.sub!( COMMENT_CHAR, '' ) if accept_comment
+      line.chomp!( RECORD_SEPARATOR )
+    end
+
+    def load_header_info
+      # File
+      file_hash = { :file => File.basename( @filename ) }
+      @header_info.merge! ( file_hash )
+
+      # Columns
+      line = read_line(true)
+      column_hash = { :columns => line.split( FIELD_SEPARATOR ) }
+      @header_info.merge! ( column_hash )
+
+      # Primary keys
+      line = read_line(true).sub!( 'primaryKey:', '' )
+      primary_hash = { :primary_keys => line.split( FIELD_SEPARATOR ) }
+      @header_info.merge! ( primary_hash )
+
+      # DB types
+      line = read_line(true).sub!( 'dbTypes:', '' )
+      primary_hash = { :db_types => line.split( FIELD_SEPARATOR ) }
+      @header_info.merge! ( primary_hash )
+
+      # Export type
+      line = read_line(true).sub!( 'exportMode:', '' )
+      primary_hash = { :export_type => line.split( FIELD_SEPARATOR ) }
+      @header_info.merge! ( primary_hash )
+    end
+
+
+    def load_footer_info
+      @file.seek(-40, IO::SEEK_END)
+      records = @file.read.split( COMMENT_CHAR ).last.chomp!( RECORD_SEPARATOR ).sub( 'recordsWritten:', '' )
+      records_hash = { :records => records }
+      @footer_info.merge! ( records_hash )
+      @file.rewind
+      @footer_info
+    end
+  end
+end

data/lib/apple_epf/version.rb

@@ -0,0 +1,3 @@
+module AppleEpf
+  VERSION = "1.0.0"
+end

data/lib/core_ext/array.rb

@@ -0,0 +1,30 @@
+# activesupport/lib/active_support/core_ext/array/extract_options.rb
+class Hash
+  # By default, only instances of Hash itself are extractable.
+  # Subclasses of Hash may implement this method and return
+  # true to declare themselves as extractable. If a Hash
+  # is extractable, Array#extract_options! pops it from
+  # the Array when it is the last element of the Array.
+  def extractable_options?
+    instance_of?(Hash)
+  end
+end
+
+class Array
+  # Extracts options from a set of arguments. Removes and returns the last
+  # element in the array if it's a hash, otherwise returns a blank hash.
+  #
+  #   def options(*args)
+  #     args.extract_options!
+  #   end
+  #
+  #   options(1, 2)           # => {}
+  #   options(1, 2, :a => :b) # => {:a=>:b}
+  def extract_options!
+    if last.is_a?(Hash) && last.extractable_options?
+      pop
+    else
+      {}
+    end
+  end
+end

data/lib/core_ext/module.rb

@@ -0,0 +1,63 @@
+# activesupport/lib/active_support/core_ext/module/attribute_accessors.rb
+class Module
+  def mattr_reader(*syms)
+    options = syms.extract_options!
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+        @@#{sym} = nil unless defined? @@#{sym}
+
+        def self.#{sym}
+          @@#{sym}
+        end
+      EOS
+
+      unless options[:instance_reader] == false || options[:instance_accessor] == false
+        class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+          def #{sym}
+            @@#{sym}
+          end
+        EOS
+      end
+    end
+  end
+
+  def mattr_writer(*syms)
+    options = syms.extract_options!
+    syms.each do |sym|
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+        def self.#{sym}=(obj)
+          @@#{sym} = obj
+        end
+      EOS
+
+      unless options[:instance_writer] == false || options[:instance_accessor] == false
+        class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+          def #{sym}=(obj)
+            @@#{sym} = obj
+          end
+        EOS
+      end
+    end
+  end
+
+  # Extends the module object with module and instance accessors for class attributes,
+  # just like the native attr* accessors for instance attributes.
+  #
+  #  module AppConfiguration
+  #    mattr_accessor :google_api_key
+  #    self.google_api_key = "123456789"
+  #
+  #    mattr_accessor :paypal_url
+  #    self.paypal_url = "www.sandbox.paypal.com"
+  #  end
+  #
+  #  AppConfiguration.google_api_key = "overriding the api key!"
+  #
+  # To opt out of the instance writer method, pass :instance_writer => false.
+  # To opt out of the instance reader method, pass :instance_reader => false.
+  # To opt out of both instance methods, pass :instance_accessor => false.
+  def mattr_accessor(*syms)
+    mattr_reader(*syms)
+    mattr_writer(*syms)
+  end
+end

data/lib/tasks/apple_epf_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :apple_epf do
+#   # Task goes here
+# end

data/spec/lib/apple_epf/downloader_spec.rb

@@ -0,0 +1,220 @@
+# encoding: UTF-8
+require File.expand_path('../../../spec_helper', __FILE__)
+
+describe AppleEpf::Downloader do
+
+  let(:type) { 'incremental' }
+  let(:filedate) { Date.parse('17-01-2013') }
+  let(:file) { 'popularity' }
+  let(:downloader) {AppleEpf::Downloader.new(type, file, filedate)}
+  let(:file_exists) { true }
+
+  describe "#get_filename_by_date_and_type" do
+    before do
+      downloader.stub(:file_exists?){ file_exists }
+    end
+
+    it "should raise exception if path can not be determined" do
+      downloader.type = 'crazytype'
+      expect {
+        downloader.get_filename_by_date_and_type
+      }.to raise_exception
+    end
+
+    context "type is full" do
+      let(:type) { 'full' }
+
+      context "and file exists" do
+        let(:file_exists) { true }
+
+        it "should return valid url if file exists" do
+          downloader.filedate = Date.parse('17-01-2013')
+          downloader.get_filename_by_date_and_type.should == "20130116/popularity20130116.tbz"
+        end
+      end
+
+      context "and file does not exists" do
+        let(:file_exists) { false }
+
+        it "should raise exception" do
+          downloader.filedate = Date.parse('17-01-2013')
+          expect {
+            downloader.get_filename_by_date_and_type
+          }.to raise_exception(AppleEpf::FileNotExist)
+        end
+      end
+
+    end
+
+    context "type is incremental" do
+      let(:type) { 'incremental' }
+
+      context "and file exists" do
+        let(:file_exists) { true }
+
+        it "should return valid url if file exists" do
+          downloader.filedate = Date.parse('17-01-2013')
+          downloader.get_filename_by_date_and_type.should == "20130109/incremental/20130117/popularity20130117.tbz"
+        end
+      end
+
+      context "and file does not exists" do
+        let(:file_exists) { false }
+
+        it "should raise exception" do
+          downloader.filedate = Date.parse('17-01-2013')
+          expect {
+            downloader.get_filename_by_date_and_type
+          }.to raise_exception(AppleEpf::FileNotExist)
+        end
+      end
+
+    end
+
+    context "type is file" do
+      pending
+    end
+
+  end
+
+  describe "#main_dir_date" do
+
+    context "full" do
+      let(:type) { 'full' }
+      it "should return the same week wednesday" do
+        downloader.filedate = Date.parse('17-01-2013') #thursday
+        downloader.send(:main_dir_date).should == "20130116"
+      end
+      # it "should return wednesday of this week if filedate is thur-sun" do
+      #   downloader.filedate = Date.parse('17-01-2013') #thursday
+      #   downloader.send(:main_dir_date).should == "20130116"
+
+      #   downloader.filedate = Date.parse('19-01-2013') #sut
+      #   downloader.send(:main_dir_date).should == "20130116"
+      # end
+
+      # it "should return wednesday of prev week if filedate is mon-wed" do
+      #   downloader.filedate = Date.parse('21-01-2013') #monday
+      #   downloader.send(:main_dir_date).should == "20130123"
+
+      #   downloader.filedate = Date.parse('23-01-2013') #wednesday
+      #   downloader.send(:main_dir_date).should == "20130123"
+      # end
+    end
+
+    context "incremental" do
+      let(:type) { 'incremental' }
+      it "should return wednesday of this week if filedate is friday-sunday" do
+        downloader.filedate = Date.parse('18-01-2013') #friday
+        downloader.send(:main_dir_date).should == "20130116"
+
+        downloader.filedate = Date.parse('19-01-2013') #sut
+        downloader.send(:main_dir_date).should == "20130116"
+      end
+
+      it "should return wednesday of prev week if filedate is monday-thursday" do
+        downloader.filedate = Date.parse('21-01-2013') #monday
+        downloader.send(:main_dir_date).should == "20130116"
+
+        downloader.filedate = Date.parse('24-01-2013') #thursday
+        downloader.send(:main_dir_date).should == "20130116"
+      end
+    end
+  end
+
+  describe "download" do
+    let(:filedate) { Date.parse('21-01-2013') }
+
+    before do
+      @tmp_dir = [Dir.tmpdir, 'epm_files'].join('/')
+      FileUtils.mkpath @tmp_dir
+
+      AppleEpf.configure do |config|
+        config.apple_id = 'test'
+        config.apple_password = 'test'
+        config.extract_dir = @tmp_dir
+      end
+
+      downloader.stub(:download_and_compare_md5_checksum)
+    end
+
+    it "should properly set url for download" do
+      downloader.stub(:file_exists?){ file_exists }
+      downloader.stub(:start_download)
+      downloader.download
+      downloader.apple_filename_full.should eq("http://feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz")
+    end
+
+    it "should properly set local file to store file in" do
+      downloader.stub(:file_exists?){ file_exists }
+      downloader.stub(:start_download)
+      downloader.download
+      downloader.download_to.should eq("#{@tmp_dir}/incremental/popularity20130121.tbz")
+    end
+
+    it "should download and save file" do
+      stub_request(:get, "http://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130123/popularity20130123.tbz").
+        to_return(:status => 200, :body => "Test\nWow", :headers => {})
+
+      downloader = AppleEpf::Downloader.new('full', file, filedate)
+      downloader.stub(:download_and_compare_md5_checksum)
+      downloader.stub(:file_exists?){ file_exists }
+      downloader.download
+      IO.read(downloader.download_to).should eq("Test\nWow")
+    end
+
+    it "should retry 3 times to download" do
+      pending
+    end
+
+    describe "dirpath" do
+      before do
+        downloader.stub(:file_exists?){ file_exists }
+        downloader.stub(:start_download)
+      end
+
+      it "should be able to change dir where to save files" do
+        tmp_dir = Dir.tmpdir
+        downloader.dirpath = [tmp_dir, 'whatever_path'].join('/')
+        downloader.download.should ==  "#{tmp_dir}/whatever_path/incremental/popularity20130121.tbz"
+      end
+    end
+
+    describe "#download_and_compare_md5_checksum" do
+      before do
+        downloader.unstub(:download_and_compare_md5_checksum)
+      end
+      it "should raise exception if md5 file does not match real md5 checksum of file" do
+        stub_request(:get, "http://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz").
+          to_return(:status => 200, :body => "Test\nWow", :headers => {})
+
+        stub_request(:get, "http://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz.md5").
+          to_return(:status => 200, :body => "tupo", :headers => {})
+
+        downloader.stub(:file_exists?){ file_exists }
+
+        expect {
+          downloader.download
+        }.to raise_exception(AppleEpf::Md5CompareError)
+
+      end
+
+      it "should not raise exception if md5 is ok" do
+        stub_request(:get, "http://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz").
+          to_return(:status => 200, :body => "Test\nWow", :headers => {})
+
+        stub_request(:get, "http://test:test@feeds.itunes.apple.com/feeds/epf/v3/full/20130116/incremental/20130121/popularity20130121.tbz.md5").
+          to_return(:status => 200, :body => "MD5 (popularity20130116.tbz) = 0371a79664856494e840af9e1e6c0152\n", :headers => {})
+
+
+        downloader.stub(:file_exists?){ file_exists }
+
+        expect {
+          downloader.download
+        }.not_to raise_exception(AppleEpf::Md5CompareError)
+
+      end
+    end
+  end
+
+end

data/spec/lib/apple_epf/exctractor_spec.rb

@@ -0,0 +1,72 @@
+# encoding: UTF-8
+require File.expand_path('../../../spec_helper', __FILE__)
+
+describe AppleEpf::Extractor do
+  let(:file_basename) { 'itunes20130111.tbz' }
+  let(:files_to_extract) { ['application', 'test_file.txt'] }
+
+  before do
+    @tmp_dir = [Dir.tmpdir, 'test_epm_files'].join('/')
+    FileUtils.mkpath @tmp_dir
+
+    AppleEpf.configure do |config|
+      config.apple_id = 'test'
+      config.apple_password = 'test'
+      config.extract_dir = @tmp_dir
+    end
+
+    @copy_to = "#{@tmp_dir}/#{file_basename}"
+    FileUtils.copy_file(apple_epf_inc_filename(file_basename), @copy_to)
+  end
+
+  after do
+    FileUtils.remove_dir(@tmp_dir)
+  end
+
+  describe "initialize" do
+    it "should set instance variables" do
+      extractor = AppleEpf::Extractor.new(@copy_to, files_to_extract)
+
+      extractor.filename.should == @copy_to
+      extractor.dirname.should == @tmp_dir
+      extractor.basename.should == file_basename
+    end
+  end
+
+  describe "perform" do
+    it "should raise error if extracting was not successful" do
+      files_to_extract = ['application', 'wrong_file.txt']
+      extractor = AppleEpf::Extractor.new(@copy_to, files_to_extract)
+
+      expect {
+        extractor.perform
+      }.to raise_exception ("Unable to extract files '#{files_to_extract.join(' ')}' from #{@copy_to}")
+    end
+
+    it "should return list if extracted files" do
+      extractor = AppleEpf::Extractor.new(@copy_to, files_to_extract)
+      extractor.perform
+      extractor.file_entry.tbz_file.should == @copy_to
+
+      expected_extracted = files_to_extract.map do |f|
+        File.join(@tmp_dir, 'itunes20130111', f)
+      end
+
+      extractor.file_entry.extracted_files.should == Hash[files_to_extract.zip(expected_extracted)]
+      extractor.file_entry.tbz_file.should == @copy_to
+    end
+
+    it "should remove file if successfully untarred" do
+      extractor = AppleEpf::Extractor.new(@copy_to, files_to_extract)
+      extractor.perform
+      File.exists?(extractor.filename).should be_false
+    end
+
+    it "should not remove file if successfully untarred and it was asked to leave file" do
+      extractor = AppleEpf::Extractor.new(@copy_to, files_to_extract)
+      extractor.keep_tbz_after_extract = true
+      extractor.perform
+      File.exists?(extractor.filename).should be_true
+    end
+  end
+end