bio-biostars-analytics 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5cff7f6e3a1b295042b32eba00069abbe877a8b9
+  data.tar.gz: a0dafd42a7a80aeb6f99a1ff1c43490d1da10507
+SHA512:
+  metadata.gz: 52b94bc235a8be6974c1f98a62da8f28bb19daf38fbc57398121333a81f5ce350fdc08d3fe78e973795df827e083254de0b648c0b1cede787978d7a6c1bcc936
+  data.tar.gz: e343dc4bfcc04d69c0fa1fba08553dfa12e0270fb0cd6bef75b689e313e84fb539c3ef933153aa7168795ebad1c9e8656513b71ce225c6961f5cc6f34b18ae1e

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - jruby-19mode # JRuby in 1.9 mode
+
+# - rbx-19mode
+#  - 1.8.7
+#  - jruby-18mode # JRuby in 1.8 mode
+#  - rbx-18mode
+
+# uncomment this line if your project needs to run something other than `rake`:
+# script: bundle exec rspec spec

data/Gemfile

@@ -0,0 +1,20 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "jeweler", "~> 2.0.1", :git => "https://github.com/technicalpickles/jeweler.git"
+  gem "bundler", ">= 1.0.21"
+  gem "bio", ">= 1.4.2"
+  gem "rdoc", "~> 3.12"
+
+  # Required for mining:
+  gem "hpricot", "~> 0.8.6"
+  gem "chronic", "~> 0.10.2"
+  gem "json", "~> 1.8.0"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Joachim Baran
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# bio-biostars-analytics
+
+[![Build Status](https://secure.travis-ci.org/joejimbo/bioruby-biostars-analytics.png)](http://travis-ci.org/joejimbo/bioruby-biostars-analytics)
+
+Data-mining analysis that make use of this gem (newest to oldest):
+
+-  [Uh-oh, Biostar: Three Years of User Metrics Analysis](http://joachimbaran.wordpress.com/2013/03/15/uh-oh-biostar/)
+-  [BioStar: Activity of its BioStars](http://joachimbaran.wordpress.com/2012/03/20/biostar-activity-of-its-biostars/)
+-  [BioStar: Is the BioStar fading? An Annual Follow-Up](http://joachimbaran.wordpress.com/2012/03/11/biostar-second-analysis/)
+-  [BioStar: Is the BioStar fading?](http://joachimbaran.wordpress.com/2011/03/07/biostar-fading/)
+
+## Installation
+
+Biostars analytics can be installed as a Ruby gem:
+
+    gem install bio-biostars-analytics
+
+Statistical analytics requires the installation of [R](http://www.r-project.org) 2.15.0 or later; requires the installation
+of the plyr package 2.15.1 or later.
+
+## Usage
+
+Data-mining: crawl the Biostars forum and retrieve data from the Biostar RESTful API; parameters
+as of March 2014:
+
+    biostars-analytics 96000 54
+
+This will create two files: `<date>_api.tsv` and `<date>_crawled.tsv`
+
+Various plots in PNG file format can be generated via:
+
+    biostar_api_stats <date>_api.tsv
+    biostar_crawled_stats <date>_crawled.tsv
+
+### Command Line Usage Instructions
+
+#### Data-Mining
+
+    Usage: biostars-analytics max_post_number months_look_back [min_post_number]
+    
+    Required parameters:
+      max_post_number    : highest number (ID) of the post that should
+                           be mined for data; the crawler will go over
+                           posts min_post_number to max_post_number
+      months_look_back   : how many months back should queries to the
+                           Biostar API go (1 month = 30 days); default
+                           value is 1
+    
+    Optional parameters:
+      min_post_number    : lowest number (ID) of the post that should
+                           be mined for data
+    
+    Output (date matches the script\'s invokation):
+      <date>_crawled.tsv : data mined from crawling over posts
+      <date>_api.tsv     : data extracted from the Biostar API
+    
+    Example: mining Biostars in March 2014:
+      biostars-analytics 96000 54
+
+#### Statistics (based on RESTful API data)
+
+    Usage: biostar_api_stats apitsvfile
+    
+    Example (data provided at http://github.com/joejimbo/bioruby-biostars-analytics):
+      biostar_api_stats data/20140328_api.tsv
+
+#### Statistics (based on forum mining/crawling)
+
+    Usage: biostar_crawled_stats crawledtsvfile
+    
+    Example (data provided at http://github.com/joejimbo/bioruby-biostars-analytics):
+      biostar_api_stats data/20140328_crawled.tsv
+
+## Project Repository
+
+Contributions can be made to the open repository on GitHub:
+
+  [http://github.com/joejimbo/bioruby-biostars-analytics](http://github.com/joejimbo/bioruby-biostars-analytics)
+
+The BioRuby community is on IRC server: irc.freenode.org, channel: #bioruby.
+
+## Cite
+
+If you use this software, please cite one of
+  
+* [BioRuby: bioinformatics software for the Ruby programming language](http://dx.doi.org/10.1093/bioinformatics/btq475)
+* [Biogem: an effective tool-based approach for scaling up open source software development in bioinformatics](http://dx.doi.org/10.1093/bioinformatics/bts080)
+
+## Biogems.info
+
+This Biogem is published at (http://biogems.info/index.html#bio-biostars-analytics)
+
+## Copyright
+
+Copyright (c) 2014 Joachim Baran. See LICENSE.txt for further details.
+

data/README.rdoc

@@ -0,0 +1,48 @@
+= bio-biostars-analytics
+
+{<img
+src="https://secure.travis-ci.org/joejimbo/bioruby-biostars-analytics.png"
+/>}[http://travis-ci.org/#!/joejimbo/bioruby-biostars-analytics]
+
+Full description goes here
+
+Note: this software is under active development!
+
+== Installation
+
+        gem install bio-biostars-analytics
+
+== Usage
+
+== Developers
+
+To use the library 
+
+        require 'bio-biostars-analytics'
+
+The API doc is online. For more code examples see also the test files in
+the source tree.
+        
+== Project home page
+
+Information on the source tree, documentation, issues and how to contribute, see
+
+  http://github.com/joejimbo/bioruby-biostars-analytics
+
+The BioRuby community is on IRC server: irc.freenode.org, channel: #bioruby.
+
+== Cite
+
+  If you use this software, please cite one of
+ 
+* [BioRuby: bioinformatics software for the Ruby programming language](http://dx.doi.org/10.1093/bioinformatics/btq475)
+* [Biogem: an effective tool-based approach for scaling up open source software development in bioinformatics](http://dx.doi.org/10.1093/bioinformatics/bts080)
+
+== Biogems.info
+
+This Biogem is published at http://biogems.info/index.html#bio-biostars-analytics
+
+== Copyright
+
+Copyright (c) 2014 Joachim Baran. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,46 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = 'bio-biostars-analytics'
+  gem.homepage = 'http://github.com/joejimbo/bioruby-biostars-analytics'
+  gem.license = 'MIT'
+  gem.summary = %Q{Biostars data-mining and statistical analysis.}
+  gem.description = %Q{Ruby script for data-mining biostars.org using web-crawling techniques as well as utilizing the Biostars RESTful API. Statistical analysis requires R (http://www.r-project.org).}
+  gem.email = 'joachim.baran@gmail.com'
+  gem.authors = [ 'Joachim Baran' ]
+  gem.executables = [ 'biostars-analytics', 'biostar_api_stats', 'biostar_crawled_stats' ]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-biostars-analytics #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/biostar_api_stats

@@ -0,0 +1,260 @@
+#!/usr/bin/ruby
+
+analysis_script = <<EOR
+args <- commandArgs(TRUE)
+
+input_tsv_file <- 'INPUT_FILE_NAME'
+
+bioeval <- function(base, column, xlabel, ylabel, colour, scale, average=TRUE) {
+    biolevels <- levels(factor(base$age_in_month))
+    avg <- rep(0, times=length(biolevels))
+    for (i in 1:length(biolevels)) {
+        avg[i] <- mean(
+            column[ base$age_in_month == biolevels[i] ]
+        )
+    }
+
+    plot(
+            base$age_in_month,
+            t(column),
+            col=colour,
+            pch=19,
+            xlab=xlabel,
+            ylab=ylabel,
+            axes=FALSE,
+            ylim=c(min(column), max(column)/scale) # Scale, so that outliers don't squish the important details.
+        )
+    if (average) {
+        lines(biolevels, avg, col=rgb(.3,.3,.3))
+    }
+    # Trendline over all data:
+    abline(lm(column ~ base$age_in_month), col=rgb(0,.9,1.0), lwd=6)
+    # Trendline over the last 24 months:
+    lookback <- 24
+    lookback_months <- biostar$age_in_month[biostar$age_in_month > (max(biostar$age_in_month - lookback))]
+    trend <- lm(tail(column, length(lookback_months)) ~ lookback_months)
+    trend_prediction <- predict(trend, newdata=data.frame(x=lookback_months))
+    lines(x=lookback_months, y=trend_prediction, col=rgb(1.0,0,.6), lwd=6)
+    
+    # Draw axis:
+    axis(1,min(base$age_in_month):max(base$age_in_month))
+    axis(2,min(column):max(column))
+}
+
+biobarplot <- function(userstats, age, max_y, xlabel, ylabel, colour) {
+    barplot(
+            userstats,
+            col=colour,
+            xlab=xlabel,
+            ylab=ylabel,
+            ylim=c(0,max_y)
+        )
+}
+
+biostar <- read.table(
+        input_tsv_file,
+        sep="\t",
+        encoding="UTF-8",
+        row.names=NULL,
+        comment.char="",
+        col.names=c(
+            "age",
+            "date",
+            "year",
+            "month",
+            "day",
+            "new_posts_in_category_1",
+            "new_posts_in_category_2",
+            "new_posts_in_category_3",
+            "new_posts_in_category_4",
+            "new_posts_in_category_5",
+            "new_posts_in_category_6",
+            "new_posts_in_category_7",
+            "new_posts_in_category_8",
+            "new_posts_in_category_9",
+            "new_posts_in_category_10",
+            "new_posts_in_category_11",
+            "new_posts_in_category_12",
+            "new_posts_in_category_13",
+            "new_posts_in_category_14",
+            "new_posts_in_category_15",
+            "new_posts_in_category_16",
+            "new_votes_of_type_accept",
+            "new_votes_of_type_bookmark",
+            "new_votes_of_type_downvote",
+            "new_votes_of_type_upvote",
+            "new_posts",
+            "new_votes",
+            "new_users",
+            "posters",
+            "poster_ages",
+            "root_post_ages",
+            "vote_post_ages",
+            "biostarbabies",
+            "empty"
+        )
+    )
+
+biostar$age_in_month <- (
+        (
+            biostar$year -
+            rep(min(biostar$year), times=length(biostar$year))
+        )*12 +
+        biostar$month
+    )
+biostar$age_in_month <- biostar$age_in_month -
+            rep(min(biostar$age_in_month), times=length(biostar$age_in_month)) +
+            1
+
+# Which users were active in a month?
+users_per_month <- lapply(unique(sort(biostar$age_in_month)), function (age_in_month) { unique(sort(sapply(strsplit(paste(biostar[biostar$age_in_month == age_in_month, ]$posters, collapse = ","), split = ","), as.integer))) })
+
+# How many users were active in a month?
+userfreq <- rep(0, max(biostar$age_in_month))
+for (age_in_month in min(biostar$age_in_month):max(biostar$age_in_month)) {
+    userfreq[age_in_month] <- length(unlist(users_per_month[age_in_month]))
+}
+
+# Determine the number of months for which users have been active.
+#  1. over the whole time span
+#  2. except for the last three months
+useractivity <- table(as.numeric(table(unlist(users_per_month))))
+useractivity_wo_last_3_months <- table(as.numeric(table(unlist(users_per_month[c(seq(4, max(length(users_per_month))))]))))
+
+png("api_participation.png", height=900, width=1300, unit="px", pointsize=26)
+biobarplot(
+        useractivity,
+        max(biostar$age_in_month),
+        (floor(useractivity[1] / 1000) + 1) * 1000,
+        'Month of Participation',
+        'Number of Active Users',
+        "#ee6633"
+    )
+dev.off()
+
+png("api_participation_wo_last_3_months.png", height=900, width=1300, unit="px", pointsize=26)
+biobarplot(
+  useractivity_wo_last_3_months,
+  max(biostar$age_in_month),
+  (floor(useractivity_wo_last_3_months[1] / 1000) + 1) * 1000,
+  'Month of Participation',
+  'Number of Active Users',
+  "#ee6633"
+)
+dev.off()
+
+userfreq_table <- as.table(userfreq)
+rownames(userfreq_table) <- seq(length(userfreq))
+png("api_users.png", height=900, width=1300, unit="px", pointsize=26)
+biobarplot(
+  userfreq_table,
+  max(biostar$age_in_month),
+  600,
+  'Biostar Month',
+  'Number of Active Users',
+  "#ff2233"
+)
+dev.off()
+
+categories_images = c(
+        "api_category_1.png",
+        "api_category_2.png",
+        "api_category_3.png",
+        "api_category_4.png",
+        "api_category_5.png",
+        "api_category_6.png",
+        "api_category_7.png",
+        "api_category_8.png",
+        "api_category_9.png",
+        "api_category_10.png",
+        "api_category_11.png",
+        "api_category_12.png",
+        "api_category_13.png",
+        "api_category_14.png",
+        "api_category_15.png",
+        "api_category_16.png",
+        "api_upvotes.png",
+        "api_downvotes.png",
+        "api_bookmarks.png",
+        "api_accepts.png"
+    )
+categories_values = list(
+        biostar$new_posts_in_category_1,
+        biostar$new_posts_in_category_2,
+        biostar$new_posts_in_category_3,
+        biostar$new_posts_in_category_4,
+        biostar$new_posts_in_category_5,
+        biostar$new_posts_in_category_6,
+        biostar$new_posts_in_category_7,
+        biostar$new_posts_in_category_8,
+        biostar$new_posts_in_category_9,
+        biostar$new_posts_in_category_10,
+        biostar$new_posts_in_category_11,
+        biostar$new_posts_in_category_12,
+        biostar$new_posts_in_category_13,
+        biostar$new_posts_in_category_14,
+        biostar$new_posts_in_category_15,
+        biostar$new_posts_in_category_16,
+        biostar$new_votes_of_type_upvote,
+        biostar$new_votes_of_type_downvote,
+        biostar$new_votes_of_type_bookmark,
+        biostar$new_votes_of_type_accept
+    )
+categories_labels = c(
+        "Questions per Day",
+        "Answers per Day",
+        "Comments per Day",
+        "Tutorials per Day",
+        "Blogs Posts per Day",
+        "Forums Posts per Day",
+        "News per Day",
+        " -- Unknown, sorry, no time to look it up now -- ",
+        "Tool Announcements per Day",
+        "FixMes per Day",
+        "Videos per Day",
+        "Job Postings per Day",
+        "Research Papers per Day",
+        "Tips per Day",
+        "Polls per Day",
+        "Ads per Day",
+        "Upvotes per Day",
+        "Downvotes per Day",
+        "Bookmarks per Day",
+        "Accepts per Day"
+    )
+for (category in seq(length(categories_images))) {
+    png(categories_images[category], height=900, width=1300, unit="px", pointsize=26)
+    bioeval(
+            biostar,
+            unlist(categories_values[category]),
+            'Biostar Month',
+            categories_labels[category],
+            rgb(100,100,0,20,maxColorValue=255),
+            2
+        )
+    dev.off()
+}
+EOR
+
+R = '/usr/bin/R'
+
+unless File.exist?(R) then
+  puts 'Please install R (http://www.r-project.org) as: /usr/bin/R'
+  puts ''
+  puts 'Also, install the plyr package via: install.packages("plyr")'
+  exit 1
+end
+
+if ARGV.length != 1 then
+  puts 'Usage: biostar_api_stats apitsvfile'
+  puts ''
+  puts 'Example (data provided at http://github.com/joejimbo/bioruby-biostars-analytics):'
+  puts '  biostar_api_stats data/20140328_api.tsv'
+  exit 2
+end
+
+IO.popen("#{R} --no-save", 'w') { |io|
+  io.puts(analysis_script.sub('INPUT_FILE_NAME', ARGV[0]))
+  io.close_write
+}
+