zipfy 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0da3f0c6e6311a52b646c2e0e4bb3e1e9a6558f3
+  data.tar.gz: 99d62a3cacf7666e7b0328ee81e38d70bf88f8b4
+SHA512:
+  metadata.gz: 97a2b41f8e28e30ebdd616ee42eff3bd684d7b013ceefd556a69a725f42b88903513ed6616ca2c72cc359d8a6afe43b382bec44553661a87303330913a1e82e2
+  data.tar.gz: bd0cd82f14544ba82f80b81555bce751109b6779f7c47116f108ae61eb7722b2408c933d404a1c12b554021d5844f86b0cb3e9121f0be696ee0b3e6de9b59c58

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/bin/test.txt

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.3
+before_install: gem install bundler -v 1.13.7

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in zipfy.gemspec
+gemspec
+gem 'putsplus'
+gem 'getoptlong'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 bjubes
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# Zipfy
+
+Zipfy calculates word distribution in text files or strings and their deviation from Zipf's law"
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'zipfy'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install zipfy
+
+## Usage
+
+Instanciate an instance of the class `Zipf` and use its instance methods, or just run Zipfy via the command line
+`zipfy file_to_analyze.txt -v`
+
+use `zipfy --help` for all possible arguments.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/bjubes/zipfy. Please ensure all pull requests are compatible with the license.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,12 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+require 'getoptlong'
+
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "zipfy"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/zipfy

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+
+#using http://robdodson.me/how-to-write-a-command-line-ruby-gem/
+require 'getoptlong'
+require 'zipfy'
+include Zipfy
+
+verbose = false
+save_path = nil
+overwrite_save = false
+
+opts = GetoptLong.new(
+    [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+    [ '--save', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+    [ '--verbose', '-v', GetoptLong::NO_ARGUMENT ]
+)
+
+file_path =  ARGV[0]
+if !file_path 
+    puts "Please specify a file to evaluate. Run zipf -h for more information"
+    exit
+end
+
+opts.each do |opt, arg|
+    case opt
+    when '--help'
+        puts <<-EOF
+Usage: zipf [FILE]
+
+FILE:
+    the file the analyze
+
+-h, --help:
+   show help
+
+-v, --verbose:
+   show table of calculations
+
+-s, --save:
+    save output of verbose to a file.
+
+-f --force:
+    overwrite save file if one already exists
+EOF
+exit 
+    when '--verbose'
+        verbose = true
+    when '--save'
+        save_path = arg
+    when '--force'
+        overwrite_save = true
+    end
+end
+
+z = Zipf.new
+z.create_distribution( z.load_file file_path)
+z.distribution
+z.sort_distribution
+z.calculate_zipfness
+z.puts_distribution if verbose
+z.save_dist_to_file save_path, overwrite_save if save_path
+puts z.calculate_std_dev_from_reg

data/lib/zipfy.rb

@@ -0,0 +1,109 @@
+require 'pry'
+# a program for seeing the zipfian distribution of a text.
+require 'putsplus'
+include Putsplus
+require_relative 'zipfy/word_data.rb'
+
+module Zipfy
+    class Zipf 
+        TAB = "\t"
+        attr_reader :distribution
+        attr_reader :total
+        attr_reader :zipf_constant
+
+        #Parses a file into an array of words
+        def load_file file_path
+            words = []
+            File.open(file_path, 'r') do |f|
+                words = f.read.to_s.split(/[\s,-]/)
+            end
+            words.each do |word|
+                word.gsub!(/\W+/, '') #remove non characters
+                word.downcase!
+            end
+            words.delete("") #remove blank entries
+            words
+        end
+
+        #
+        # sets instance var distribution to the distirubtion of the words set passed
+        #
+        def create_distribution words
+            temp_hash = Hash.new(0)
+            @distribution = []
+            @total = 0
+            @zipf_constant = nil
+
+            words.each do |word|
+                temp_hash[word] += 1
+                @total += 1
+            end
+            temp_hash.keys.each do |k|
+                @distribution << WordData.new(k, temp_hash[k])
+            end
+        end
+
+        def sort_distribution
+            @distribution = @distribution.sort_by {|wd| wd.frequency.to_i}
+        end
+
+        def calculate_zipfness
+            calculate_zipf_constant unless @zipf_constant
+
+            #zipf number is equal to freq * zipf_const/total
+            # so call zipf_const/total 'z'
+            z = @zipf_constant/@total.to_f
+            length = @distribution.length
+
+            @distribution.map do |d|
+               d.zip_number = z * d.frequency
+            end
+        end
+
+        def puts_distribution
+            underline " "*6 + "Word Distirbution" + " "*6
+            length = @distribution.length
+            @distribution.each_with_index do |wd, i|
+                tabs = 4 - (wd.word.length / 8).floor
+                puts wd.word + (TAB * tabs).to_s + wd.frequency.to_s + TAB + (length - i).to_s + TAB + wd.zip_number.to_s
+            end
+        end
+
+        def save_dist_to_file file_path, overwrite_file = false
+            if File.exists?(file_path) && overwrite_file
+                puts "Requires --force to overwrite existing file '#{file_path}'"
+                exit
+            end
+        end
+
+        def calculate_zipf_constant
+            #this word has rank =1 , so its zipf factor should be 1/n = 1/1 = 1
+            # so find its percentage and then multiply it up to one.
+
+            #make the following true
+            #freq of most common word / total words = 1 
+            #by adding a constant to the right side
+            # where constant = total words/ freq of most common word
+
+            @zipf_constant = @total.to_f / @distribution.last.frequency.to_f
+            @zipf_constant
+        end
+
+        #must calculate zipfness first
+        def calculate_std_dev_from_reg
+            #the zipfness should be equal to 1/rank. lets see the average deviatoin form that number
+            length = @distribution.length
+            deviations = []
+
+            @distribution.each_with_index do |wd,i|
+                theoretical = 1/(length - i).to_f
+                actual = wd.zip_number
+                deviation = (theoretical - actual).abs
+                deviations << deviation
+            end
+            sum = 0.0
+            deviations.each {|d| sum += d}
+            (sum/length.to_f)
+         end
+    end
+end

data/lib/zipfy/version.rb

@@ -0,0 +1,3 @@
+module Zipfy
+  VERSION = "1.0.0"
+end

data/lib/zipfy/word_data.rb

@@ -0,0 +1,13 @@
+module Zipfy
+	class WordData
+
+		attr_accessor :word
+		attr_accessor :frequency
+		attr_accessor :zip_number # theoretically equal to 1/rank in a text
+
+		def initialize word, frequency = 1
+			@word = word
+			@frequency = frequency
+		end
+	end
+end

data/zipfy.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'zipfy/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "zipfy"
+  spec.version       = Zipfy::VERSION
+  spec.authors       = ["bjubes"]
+  spec.email         = ["brian2386@gmail.com"]
+
+  spec.summary       = "Zipfy calculates word distribution in text files or strings and their deviation from Zipf's law"
+  spec.homepage      = "https://github.com/bjubes/ziphy"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "bin"
+  spec.executables   = ["zipfy"]
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'putsplus'
+  spec.add_development_dependency "bundler", "~> 1.13"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: zipfy
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- bjubes
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: putsplus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: 
+email:
+- brian2386@gmail.com
+executables:
+- zipfy
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bin/zipfy
+- lib/zipfy.rb
+- lib/zipfy/version.rb
+- lib/zipfy/word_data.rb
+- zipfy.gemspec
+homepage: https://github.com/bjubes/ziphy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Zipfy calculates word distribution in text files or strings and their deviation
+  from Zipf's law
+test_files: []