movieDB 0.1.10 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d50cca9ba4a8eab49599c8fd2dbe0ab856993d34
-  data.tar.gz: 0f8f434ce0c78e6b4604cc326717b75373777818
+  metadata.gz: 4d6f86e45dd6df3973448eb08ac5e76258be9fe6
+  data.tar.gz: 740512f4b3c575ca884148584b24eaae6b62ea05
 SHA512:
-  metadata.gz: aba42600c6385faa6383be3e23eae5fa604349de78c2d607d38d6ac34ceaf91239d14db5bb652f14c16c879973acba335617ef9e3e9537bb8e6b46af9e0ee713
-  data.tar.gz: 7b2f064cd6dbc380a1f798923b6ba6428af2b4d161a3421c92eaab1d348a636d5419c17a75577a350fe075133f004f98bc7c456567bc22968c14f4ebdb3c53d6
+  metadata.gz: 8416c049afe77426e2fd6daf29b91e0f60528cf24309effd09d072c10330a9f1d5fa27bf8d59eee2029b4be805ccf25d5253c4205dad1c485c738fce253698c1
+  data.tar.gz: 23f5c489154e0c4ce12452056cd4a5a5465de77df80fa26a21e74bfb9eefc85bbfb09922f6bb728411737016bed2d44980ba93bdede2c7929daf782981ce0a79

data/README.md

@@ -47,10 +47,99 @@ The exported movie data is stored in your reports directory.
 
     $ cd /reports/imdb_raw_data_20131216.xls
 
-## Usage - Data Analysis
+## Usage - Analysing Data and Generating Stats
 
     $ irb
 
     > require 'MovieDB/data_analysis'
 
-    > "work-in-progress"
+    > require 'MovieDB/data_process'
+
+    > MovieDB::DataProcess.send(:basic_statistics, 'imdb_raw_data_20131216.xls')
+
+## Exported - Analyzed Data 
+
+The exported analyzed data is stored in your reports directory.
+
+    $ cd /reports/basic_statistic_20131216.xls
+
+## What's Next
+
+ More statistical computations coming. This includes:
+
+   > Gauss_Newton_Algorithm 
+        > Iteratively_Reweighted_Least_Squares 
+        > Lack_Of_Fit_Sum_Of_Squares 
+        > Least_Squares_Support_Vector_Machine 
+        > Mean_Squared_Error 
+        > Moving_Least_Sqares 
+        > Non_Linear_Iterative_Partial_Least_Squares 
+        > Non_Linear_Least_Squares 
+        > Ordinary_Least_Squares 
+        > Partial_Least_Squares_Regression 
+        > Partition_Of_Sums_Of_Squares 
+        > Proofs_Involving_Ordinary_Least_Squares 
+        > Residual_Sum_Of_Squares 
+        > Total_Least_Squares 
+        > Total_Sum_Of_Squares 
+      
+    > EstimationOfDensity
+      > Cluster_Weighted_Modeling 
+      > Density_Estimation 
+      > Discretization_Of_Continuous_Features 
+      > Mean_Integrated_Squared_Error 
+      > Multivariate_Kernel_Density_Estimation 
+      > Variable_Kernel_Density_Estimation 
+    
+    > ExploratoryDataAnalysis
+      > Data_Reduction 
+      > Table_Diagonalization 
+      > Configural_Frequency_Analysis 
+      > Median_Polish 
+      > Stem_And_Leaf_Display 
+    
+    > Data_Mining
+      > Applied_DataMining 
+      > Cluster_Analysis 
+      > Dimension_Reduction 
+      > Applied_DataMining 
+    
+    > RegressionAnalysis
+      > Choice_Modelling 
+
+      > Generalized_Linear_Model 
+        > Binomial_Regression         
+        > Generalized_Additive_Model         
+        > Linear_Probability_Model         
+        > Poisson_Regression         
+        > Zero_Inflated_Model            
+      
+      > Nonparametric_Regression 
+      > Statistical_Outliers 
+      > Regression_And_Curve_Fitting_Software 
+      > Regression_Diagnostics 
+      > Regression_Variable_Selection 
+      > Regression_With_Time_Series_Structure 
+      > Robust_Regression 
+      > Choice_Modeling 
+    
+    > Resampling
+      > Bootstrapping_Population 
+    
+    > Sensitivity_Analysis
+      > Variance_Based_Sensitivity_Analysis 
+      > Elementary_Effects_Method 
+      > Experimental_Uncertainty_Analysis 
+      > Fourier_Amplitude_Sensitivity_Testing 
+      > Hyperparameter 
+    
+    > Time_series_Analysis
+      > Frequency_Deviation 
+
+## Contact me
+
+If you'd like to collaborate, please feel free to fork source code on github. 
+
+Also, You can also contact me at albertmck@gmail.com
+
+     

data/lib/movieDB/data_analysis.rb

@@ -8,8 +8,9 @@ module MovieDB
   class DataAnalysis < MovieDB::Movie
     module AnalysisOfVariance
       module LeastSquares
-        module Coefficient_Of_Determination
-          def coefficient_of_determination (directory_name)
+        module Statistic
+
+          def basic_statistic (directory_name)
             open_spreadsheet(directory_name)
             perform_computation
             insert_data_to_existing_xls_file
@@ -21,25 +22,102 @@ module MovieDB
           end
 
           def perform_computation
-            @col_0 = []
 
-            @sheet.each_with_index do |row, i|
-              @col_0 << @sheet[i, 1]
-            end
+            total_columns = 17
+            @column = []
+            @row_count = @sheet.rows.count
+
+            1.upto(total_columns) do |c|
+              @column = [] # set instance variable to an empty array
+
+              ##
+              # loop through to collect all elements
+              # The returned array includes both strings and integers elements
+
+              @sheet.each_with_index do |row, i|
+                @column << @sheet[i, 0 + c ]
+              end
+
+              @column.shift # delete the string header from the array
+              @column.compact! # delete nil from the array
+              row_count = @sheet.rows.count
+
+              ##
+              # Perform computation on the data collected
+              # TODO: Need to use coefficienct statistical formula
+              # Calculate median as an example but COD formula must be used
+
+
+              if @column.all? {|i| (1..99999999).include? (i)}
+
+                n = @column.count
+                    @column.sort!
+
+                ##
+                # Mean is commonly called as average.Mean or Average is defined as the sum of 
+                # all the given elements divided by the total number of elements.
+                #
+                # Range is the difference between the highest and the lowest values in a 
+                # frequency distribution.
+                #
+                # Mode is the most frequently occurring value in a frequency distribution.
 
-            ##
-            # Perform computation on the data collected
-            # TODO: Need to use coefficienct statistical formula
-            # Calculate median as an example but COD formula must be used
+                @mean = @column.sum/n # Find the mean
+                @range = @column.max - @column.min # Find the range
 
-            drop_header = @col_0.shift
+                freq = @column.inject(Hash.new(0)) { |h, v| h[v] += 1; h }
+                @mode =  @column.sort_by { |v| freq[v]}.last # Find the mode
 
-            row_count = @sheet.rows.count
-            column_count = @sheet.columns[0].count
+                ##
+                # Calculate Standard Deviation 
+                # Standard deviation is a statistical measure of spread or variability.
+                # The standard deviation is the root mean square (RMS) deviation of the
+                # values from their arithmetic mean.
 
-            @data_processing = @col_0.inject do |sum, n|
-              (sum + n)/(row_count-1)
-            end
+                @column_squared = []
+                @column.each do |col|
+                  @column_squared << col**2
+                end
+
+                @sum_of_column = @column.sum
+                @sum_of_column_squared = @column_squared.sum
+                @standard_dev = Math.sqrt((@sum_of_column_squared -((@sum_of_column)*(@sum_of_column)/n))/(n-1))
+
+                 if n.odd?
+                   index = (n + 1)/2
+                   @median = @column[index - 1] # Subtract -1 to reduce index value since array start with an index 0.
+                 else
+                   middle_index = n/2
+                   right_index = middle_index + 1
+                   @median = (@column[middle_index - 1] + @column[right_index - 1])/2
+                 end
+
+              else
+                @median = "N/A"
+                @mean = "N/A"
+                @range = "N/A"
+                @mode = "N/A"
+                @standard_dev = "N/A"
+              end
+
+              ##
+              # Insert results into spreadsheet cell
+
+              @sheet[@row_count + 2, 0 ] =  "Mean"
+              @sheet[@row_count + 2, 0 + c ] = @mean
+
+              @sheet[@row_count + 3, 0 ] =  "Median"
+              @sheet[@row_count + 3, 0 + c ] =  @median
+
+              @sheet[@row_count + 4, 0 ] =  "Range"
+              @sheet[@row_count + 4, 0 + c ] =  @range
+
+              @sheet[@row_count + 5, 0 ] =  "Mode"
+              @sheet[@row_count + 5, 0 + c ] =  @mode
+
+              @sheet[@row_count + 6, 0 ] =  "Standard Deviation"
+              @sheet[@row_count + 6, 0 + c ] =  @standard_dev
+            end 
           end
 
           def report_name
@@ -51,14 +129,11 @@ module MovieDB
 
           def insert_data_to_existing_xls_file
             filename = ("#{report_name}.xls")
-            #@book.worksheet(0).insert_row(4, [@data_processing ])
-            @sheet[5, 1] = @data_processing
-            @sheet.row(6).push "Median", @data_processing 
-
             @book.write File.join('reports', filename)
             return filename
           end
         end
+        module Coefficient_Of_Determination; end
         module Discrete_Least_Squares_Meshless_Method; end
         module Explained_Sum_Of_Squares; end
         module Fraction_Of_Variance_Unexplained; end

data/lib/movieDB/data_process.rb

@@ -3,6 +3,7 @@ require 'MovieDB/data_analysis'
 module MovieDB
   class DataProcess
     PATH_AOV = MovieDB::DataAnalysis::AnalysisOfVariance::LeastSquares
+    extend PATH_AOV::Statistic
     extend PATH_AOV::Coefficient_Of_Determination
     include PATH_AOV::Explained_Sum_Of_Squares
     include PATH_AOV::Fraction_Of_Variance_Unexplained

data/lib/movieDB/version.rb

@@ -1,3 +1,3 @@
 module MovieDB
-  VERSION = "0.1.10"
+  VERSION = "0.2.0"
 end

data/spec/data_process_spec.rb

@@ -5,11 +5,10 @@ describe MovieDB::DataProcess do
   describe "#AnalysisOfVariance" do
    describe "#LeastSquares" do
      describe "#Coefficient_Of_Determination" do
-        let(:cod) {MovieDB::DataProcess}
-        #let(:cod) {MovieDB::DataProcess.analyze_cod(imdb_raw_data_2013121820.xls)}
+        let(:basic_stat) {MovieDB::DataProcess}
 
         it "should return the cof" do
-         cod.send(:coefficient_of_determination, 'imdb_raw_data_2013121911.xls').should == []
+         basic_stat.send(:basic_statistic, 'imdb_raw_data_2013121911.xls').should == []
         end 
 
         it "raise error if file does not exist" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: movieDB
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.2.0
 platform: ruby
 authors:
 - Albert_McKeever
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-19 00:00:00.000000000 Z
+date: 2013-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -164,10 +164,8 @@ files:
 - movieDB.gemspec
 - npm-debug.log
 - reports/.DS_Store
-- reports/Coefficient_Of_Determination_2013121918.xls
-- reports/add.xls
+- reports/Statistic_2013122017.xls
 - reports/imdb_raw_data_2013121911.xls
-- reports/imdb_raw_data_2013121912.xls
 - spec/.DS_Store
 - spec/data_export_spec.rb
 - spec/data_process_spec.rb