blackboard_feed 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: df77aaf850b61af878f09aee5271db5a6a773e38
+  data.tar.gz: 233190469af03db0a9007a67b81468f4e65b6f96
+SHA512:
+  metadata.gz: b8307ee44f0b0554d175570f4f2a79ba8ef9a5b472d682674bea59972e9f6e8af7a4a5c1cce18261f23b3a05bf9dd990860e267a7ec4efe822554a4d9257f255
+  data.tar.gz: ca4830f8cfc106c9f76fe6b71e15f55980bb7e5d3e8e1335892f1b17292512f92f7ad66159d06af8f85b97789554f6ff59acc851445d5c93e4de786551c2d9f2

data/.gitignore

@@ -0,0 +1,24 @@
+*.sql
+s
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.2.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gem 'sequel'
+gem 'mysql2'

data/README.md

@@ -0,0 +1,22 @@
+# blackboard_feed
+
+## Setup
+This program requires a few pieces of software to be installed:
+* mysql
+* ruby version >= 1.9.3
+
+This program requires 3 environment variables to be set:
+* DATABASE_USER - the user that will log into the mysql database
+* DATABASE_PASS - the password for that user
+* FEED_PATH - the absolute path to the blackboard feed files
+
+To get the databases setup please add the direcotry containing these files to your path and run (TODO):
+```bash
+feed_setup.rb
+```
+
+## Usage
+Add the directory that contains these files to your path and run:
+```bash
+update.rb
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "ruby_feed"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/feed_setup

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'ruby_feed'
+
+RubyFeed::Feed.setup

data/bin/feed_update

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'ruby_feed'
+
+RubyFeed::Feed.update

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/ruby_feed.rb

@@ -0,0 +1,22 @@
+require "ruby_feed/version"
+require "ruby_feed/sql_generator"
+require "ruby_feed/table_migrator"
+require "ruby_feed/parser"
+
+module RubyFeed
+  class Feed
+    DATABASES = {current: "current", yesterday: "yesterday", two_days_ago: "2_days_ago"}
+
+    def self.setup
+      TableMigrator.create_current(DATABASES[:current])
+      TableMigrator.create_current(DATABASES[:yesterday])
+      TableMigrator.create_current(DATABASES[:two_days_ago])
+    end
+
+    def self.update
+      TableMigrator.drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
+      TableMigrator.drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
+      TableMigrator.create_current(DATABASES[:current])
+    end
+  end
+end

data/lib/ruby_feed/parser.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+class Parser
+  FEED_FILES_PATH = "/home/jchae/feed/"
+
+  def self.parse_headers(file)
+    raw = %x{ head -1 #{FEED_FILES_PATH}#{file} }
+    parse_header(raw)
+  end
+
+  def self.parse_header(header)
+    headers = header.split("|")
+    headers.map { |h|
+      h.strip!
+      h.chomp!
+      h.downcase!
+    }
+  end
+
+  def self.parse_file(lines, db_name)
+    insert_string = ""
+    lines.each do |line|
+      insert_string << "insert into #{db_name} values ("
+      line.split("|").each do |element|
+        element.chomp!
+        insert_string << "'" << element.gsub(/'/,"\\'") << "'" << ","
+      end
+      insert_string << ")"
+    end
+    insert_string.gsub!(/,\)/, "); ")
+  end
+end

data/lib/ruby_feed/sql_generator.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require_relative './parser'
+
+class SqlGenerator
+  def self.generate_table_create_string(table_name)
+
+    create_string = "create table #{table_name} ("
+    headers = Parser.parse_headers(table_name)
+
+    headers.each { |header|
+      create_string << header << " varchar(255), "
+    }
+
+    create_string.sub!(/, $/, "")
+    create_string << ")"
+
+    create_string
+  end
+
+  def self.generate_drop(db_name)
+    "drop database #{db_name}"
+  end
+
+  def self.generate_rename(to, from)
+    "rename database '#{from}' to '#{to}'"
+  end
+
+  def self.generate_db_create(db_name)
+    "create database #{db_name}"
+  end
+end

data/lib/ruby_feed/table_migrator.rb

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+
+require 'sequel'
+require_relative './sql_generator'
+
+class ::TableMigrator
+
+  FEED_PATH = ENV["FEED_PATH"]
+  DATABASE_USER = ENV["DATABASE_USER"]
+  DATABASE_PASS = ENV["DATABASE_PASS"]
+
+  # def self.setup
+  #   create_current(DATABASES[:current])
+  #   create_current(DATABASES[:yesterday])
+  #   create_current(DATABASES[:two_days_ago])
+  # end
+
+  # def self.update
+  #   drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
+  #   drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
+  #   create_current(DATABASES[:current])
+  # end
+
+  def self.create_current(db_name)
+    drop(db_name)
+    create(db_name)
+    create_tables(db_name)
+    insert_into(db_name)
+  end
+
+  def self.insert_into(db_name)
+    db = connect(db_name)
+    table_names = get_table_names
+
+    table_names.each do |table|
+      table_name = File.basename(table)
+      p "#{FEED_PATH}#{table}"
+      table_file =  "#{FEED_PATH}#{table}"
+      insert_string = Parser.parse_file(File.readlines(table_file).drop(1), table_name)
+      next if insert_string.nil?
+      insert_string.split(";").each do |s|
+        next if s == " "
+        # p s
+        db.run(s)
+      end
+    end
+  end
+
+  def self.create_tables(db_name)
+    db = connect(db_name)
+    table_names = get_table_names
+
+    tables = table_names.map {|f| File.basename(f, ".*")}
+    non_existant_tables = tables.reject { |file| db.table_exists?(file) }
+
+    non_existant_tables.each do |table|
+      create_string = SqlGenerator.generate_table_create_string(table)
+      # puts create_string
+      db.run(create_string)
+    end
+  end
+
+  def self.migrate_info(to, from)
+    drop_and_recreate(to)
+    to = connect(to)
+    from = connect(from)
+  end
+
+  def self.drop_and_recreate(to, from)
+    drop(to)
+    create(to)
+    dump_old_to_new(to, from)
+  end
+
+  def self.dump_old_to_new(to, from)
+    `mysqldump -u #{DATABASE_USER} -p#{DATABASE_PASS} #{from} | mysql -u #{DATABASE_USER} -p#{DATABASE_PASS} #{to}`
+  end
+
+  def self.drop(db_name)
+    db = connect
+    drop_string = SqlGenerator.generate_drop(db_name)
+    db.run(drop_string)
+  end
+
+  def self.rename(to, from)
+    db = connect
+    rename_string = SqlGenerator.generate_rename(to, from)
+    p rename_string
+    db.run(rename_string)
+  end
+
+  def self.create(db_name)
+    db = connect
+    create_string = SqlGenerator.generate_db_create(db_name)
+    db.run(create_string)
+  end
+
+  private
+  def self.connect(db_name="mysql")
+    Sequel.connect("mysql2://#{DATABASE_USER}:#{DATABASE_PASS}@localhost/#{db_name}")
+  end
+
+  def self.get_table_names
+    files = Dir.entries(FEED_PATH) - entries_to_be_excluded_from_db
+    files
+  end
+
+  def self.entries_to_be_excluded_from_db
+    [".", "..", "list"]
+  end
+end
+
+# ::TableMigrator.create_tables("apples")
+# ::TableMigrator.insert_into("apples")

data/lib/ruby_feed/version.rb

@@ -0,0 +1,3 @@
+module RubyFeed
+  VERSION = "0.1.0"
+end

data/ruby_feed.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ruby_feed/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "blackboard_feed"
+  spec.version       = RubyFeed::VERSION
+  spec.authors       = ["Jacob Chae"]
+  spec.email         = ["jbcden@gmail.com"]
+
+  spec.summary       = "A feed file parser"
+  spec.description   = "This parses feed files for the Oberlin CIT office and loads them into a mysql db"
+  spec.homepage      = "https://github.com/jbcden/blackboard_feed"
+  spec.required_ruby_version = ">=1.9.3"
+  spec.license = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables = ["feed_update", "feed_setup"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: blackboard_feed
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jacob Chae
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This parses feed files for the Oberlin CIT office and loads them into
+  a mysql db
+email:
+- jbcden@gmail.com
+executables:
+- feed_update
+- feed_setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/feed_setup
+- bin/feed_update
+- bin/setup
+- lib/ruby_feed.rb
+- lib/ruby_feed/parser.rb
+- lib/ruby_feed/sql_generator.rb
+- lib/ruby_feed/table_migrator.rb
+- lib/ruby_feed/version.rb
+- ruby_feed.gemspec
+homepage: https://github.com/jbcden/blackboard_feed
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A feed file parser
+test_files: []