blackboard_feed 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: df77aaf850b61af878f09aee5271db5a6a773e38
4
+ data.tar.gz: 233190469af03db0a9007a67b81468f4e65b6f96
5
+ SHA512:
6
+ metadata.gz: b8307ee44f0b0554d175570f4f2a79ba8ef9a5b472d682674bea59972e9f6e8af7a4a5c1cce18261f23b3a05bf9dd990860e267a7ec4efe822554a4d9257f255
7
+ data.tar.gz: ca4830f8cfc106c9f76fe6b71e15f55980bb7e5d3e8e1335892f1b17292512f92f7ad66159d06af8f85b97789554f6ff59acc851445d5c93e4de786551c2d9f2
@@ -0,0 +1,24 @@
1
+ *.sql
2
+ s
3
+ *.gem
4
+ *.rbc
5
+ .bundle
6
+ .config
7
+ .yardoc
8
+ Gemfile.lock
9
+ InstalledFiles
10
+ _yardoc
11
+ coverage
12
+ doc/
13
+ lib/bundler/man
14
+ pkg
15
+ rdoc
16
+ spec/reports
17
+ test/tmp
18
+ test/version_tmp
19
+ tmp
20
+ *.bundle
21
+ *.so
22
+ *.o
23
+ *.a
24
+ mkmf.log
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.2.1
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "https://rubygems.org"
2
+
3
+ gem 'sequel'
4
+ gem 'mysql2'
@@ -0,0 +1,22 @@
1
+ # blackboard_feed
2
+
3
+ ## Setup
4
+ This program requires a few pieces of software to be installed:
5
+ * mysql
6
+ * ruby version >= 1.9.3
7
+
8
+ This program requires 3 environment variables to be set:
9
+ * DATABASE_USER - the user that will log into the mysql database
10
+ * DATABASE_PASS - the password for that user
11
+ * FEED_PATH - the absolute path to the blackboard feed files
12
+
13
+ To get the databases setup please add the direcotry containing these files to your path and run (TODO):
14
+ ```bash
15
+ feed_setup.rb
16
+ ```
17
+
18
+ ## Usage
19
+ Add the directory that contains these files to your path and run:
20
+ ```bash
21
+ update.rb
22
+ ```
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "ruby_feed"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'ruby_feed'
4
+
5
+ RubyFeed::Feed.setup
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'ruby_feed'
4
+
5
+ RubyFeed::Feed.update
@@ -0,0 +1,7 @@
1
+ #!/bin/bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+
5
+ bundle install
6
+
7
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,22 @@
1
+ require "ruby_feed/version"
2
+ require "ruby_feed/sql_generator"
3
+ require "ruby_feed/table_migrator"
4
+ require "ruby_feed/parser"
5
+
6
+ module RubyFeed
7
+ class Feed
8
+ DATABASES = {current: "current", yesterday: "yesterday", two_days_ago: "2_days_ago"}
9
+
10
+ def self.setup
11
+ TableMigrator.create_current(DATABASES[:current])
12
+ TableMigrator.create_current(DATABASES[:yesterday])
13
+ TableMigrator.create_current(DATABASES[:two_days_ago])
14
+ end
15
+
16
+ def self.update
17
+ TableMigrator.drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
18
+ TableMigrator.drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
19
+ TableMigrator.create_current(DATABASES[:current])
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,32 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ class Parser
4
+ FEED_FILES_PATH = "/home/jchae/feed/"
5
+
6
+ def self.parse_headers(file)
7
+ raw = %x{ head -1 #{FEED_FILES_PATH}#{file} }
8
+ parse_header(raw)
9
+ end
10
+
11
+ def self.parse_header(header)
12
+ headers = header.split("|")
13
+ headers.map { |h|
14
+ h.strip!
15
+ h.chomp!
16
+ h.downcase!
17
+ }
18
+ end
19
+
20
+ def self.parse_file(lines, db_name)
21
+ insert_string = ""
22
+ lines.each do |line|
23
+ insert_string << "insert into #{db_name} values ("
24
+ line.split("|").each do |element|
25
+ element.chomp!
26
+ insert_string << "'" << element.gsub(/'/,"\\'") << "'" << ","
27
+ end
28
+ insert_string << ")"
29
+ end
30
+ insert_string.gsub!(/,\)/, "); ")
31
+ end
32
+ end
@@ -0,0 +1,31 @@
1
+ #!/usr/bin/env ruby
2
+ require_relative './parser'
3
+
4
+ class SqlGenerator
5
+ def self.generate_table_create_string(table_name)
6
+
7
+ create_string = "create table #{table_name} ("
8
+ headers = Parser.parse_headers(table_name)
9
+
10
+ headers.each { |header|
11
+ create_string << header << " varchar(255), "
12
+ }
13
+
14
+ create_string.sub!(/, $/, "")
15
+ create_string << ")"
16
+
17
+ create_string
18
+ end
19
+
20
+ def self.generate_drop(db_name)
21
+ "drop database #{db_name}"
22
+ end
23
+
24
+ def self.generate_rename(to, from)
25
+ "rename database '#{from}' to '#{to}'"
26
+ end
27
+
28
+ def self.generate_db_create(db_name)
29
+ "create database #{db_name}"
30
+ end
31
+ end
@@ -0,0 +1,114 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'sequel'
4
+ require_relative './sql_generator'
5
+
6
+ class ::TableMigrator
7
+
8
+ FEED_PATH = ENV["FEED_PATH"]
9
+ DATABASE_USER = ENV["DATABASE_USER"]
10
+ DATABASE_PASS = ENV["DATABASE_PASS"]
11
+
12
+ # def self.setup
13
+ # create_current(DATABASES[:current])
14
+ # create_current(DATABASES[:yesterday])
15
+ # create_current(DATABASES[:two_days_ago])
16
+ # end
17
+
18
+ # def self.update
19
+ # drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
20
+ # drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
21
+ # create_current(DATABASES[:current])
22
+ # end
23
+
24
+ def self.create_current(db_name)
25
+ drop(db_name)
26
+ create(db_name)
27
+ create_tables(db_name)
28
+ insert_into(db_name)
29
+ end
30
+
31
+ def self.insert_into(db_name)
32
+ db = connect(db_name)
33
+ table_names = get_table_names
34
+
35
+ table_names.each do |table|
36
+ table_name = File.basename(table)
37
+ p "#{FEED_PATH}#{table}"
38
+ table_file = "#{FEED_PATH}#{table}"
39
+ insert_string = Parser.parse_file(File.readlines(table_file).drop(1), table_name)
40
+ next if insert_string.nil?
41
+ insert_string.split(";").each do |s|
42
+ next if s == " "
43
+ # p s
44
+ db.run(s)
45
+ end
46
+ end
47
+ end
48
+
49
+ def self.create_tables(db_name)
50
+ db = connect(db_name)
51
+ table_names = get_table_names
52
+
53
+ tables = table_names.map {|f| File.basename(f, ".*")}
54
+ non_existant_tables = tables.reject { |file| db.table_exists?(file) }
55
+
56
+ non_existant_tables.each do |table|
57
+ create_string = SqlGenerator.generate_table_create_string(table)
58
+ # puts create_string
59
+ db.run(create_string)
60
+ end
61
+ end
62
+
63
+ def self.migrate_info(to, from)
64
+ drop_and_recreate(to)
65
+ to = connect(to)
66
+ from = connect(from)
67
+ end
68
+
69
+ def self.drop_and_recreate(to, from)
70
+ drop(to)
71
+ create(to)
72
+ dump_old_to_new(to, from)
73
+ end
74
+
75
+ def self.dump_old_to_new(to, from)
76
+ `mysqldump -u #{DATABASE_USER} -p#{DATABASE_PASS} #{from} | mysql -u #{DATABASE_USER} -p#{DATABASE_PASS} #{to}`
77
+ end
78
+
79
+ def self.drop(db_name)
80
+ db = connect
81
+ drop_string = SqlGenerator.generate_drop(db_name)
82
+ db.run(drop_string)
83
+ end
84
+
85
+ def self.rename(to, from)
86
+ db = connect
87
+ rename_string = SqlGenerator.generate_rename(to, from)
88
+ p rename_string
89
+ db.run(rename_string)
90
+ end
91
+
92
+ def self.create(db_name)
93
+ db = connect
94
+ create_string = SqlGenerator.generate_db_create(db_name)
95
+ db.run(create_string)
96
+ end
97
+
98
+ private
99
+ def self.connect(db_name="mysql")
100
+ Sequel.connect("mysql2://#{DATABASE_USER}:#{DATABASE_PASS}@localhost/#{db_name}")
101
+ end
102
+
103
+ def self.get_table_names
104
+ files = Dir.entries(FEED_PATH) - entries_to_be_excluded_from_db
105
+ files
106
+ end
107
+
108
+ def self.entries_to_be_excluded_from_db
109
+ [".", "..", "list"]
110
+ end
111
+ end
112
+
113
+ # ::TableMigrator.create_tables("apples")
114
+ # ::TableMigrator.insert_into("apples")
@@ -0,0 +1,3 @@
1
+ module RubyFeed
2
+ VERSION = "0.1.0"
3
+ end
@@ -0,0 +1,24 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'ruby_feed/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "blackboard_feed"
8
+ spec.version = RubyFeed::VERSION
9
+ spec.authors = ["Jacob Chae"]
10
+ spec.email = ["jbcden@gmail.com"]
11
+
12
+ spec.summary = "A feed file parser"
13
+ spec.description = "This parses feed files for the Oberlin CIT office and loads them into a mysql db"
14
+ spec.homepage = "https://github.com/jbcden/blackboard_feed"
15
+ spec.required_ruby_version = ">=1.9.3"
16
+ spec.license = "MIT"
17
+
18
+ spec.files = `git ls-files -z`.split("\x0")
19
+ spec.executables = ["feed_update", "feed_setup"]
20
+ spec.require_paths = ["lib"]
21
+
22
+ spec.add_development_dependency "bundler", "~> 1.9"
23
+ spec.add_development_dependency "rake", "~> 10.0"
24
+ end
metadata ADDED
@@ -0,0 +1,91 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: blackboard_feed
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Jacob Chae
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-04-29 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.9'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.9'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ description: This parses feed files for the Oberlin CIT office and loads them into
42
+ a mysql db
43
+ email:
44
+ - jbcden@gmail.com
45
+ executables:
46
+ - feed_update
47
+ - feed_setup
48
+ extensions: []
49
+ extra_rdoc_files: []
50
+ files:
51
+ - ".gitignore"
52
+ - ".travis.yml"
53
+ - Gemfile
54
+ - Gemfile.lock
55
+ - README.md
56
+ - Rakefile
57
+ - bin/console
58
+ - bin/feed_setup
59
+ - bin/feed_update
60
+ - bin/setup
61
+ - lib/ruby_feed.rb
62
+ - lib/ruby_feed/parser.rb
63
+ - lib/ruby_feed/sql_generator.rb
64
+ - lib/ruby_feed/table_migrator.rb
65
+ - lib/ruby_feed/version.rb
66
+ - ruby_feed.gemspec
67
+ homepage: https://github.com/jbcden/blackboard_feed
68
+ licenses:
69
+ - MIT
70
+ metadata: {}
71
+ post_install_message:
72
+ rdoc_options: []
73
+ require_paths:
74
+ - lib
75
+ required_ruby_version: !ruby/object:Gem::Requirement
76
+ requirements:
77
+ - - ">="
78
+ - !ruby/object:Gem::Version
79
+ version: 1.9.3
80
+ required_rubygems_version: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - ">="
83
+ - !ruby/object:Gem::Version
84
+ version: '0'
85
+ requirements: []
86
+ rubyforge_project:
87
+ rubygems_version: 2.4.6
88
+ signing_key:
89
+ specification_version: 4
90
+ summary: A feed file parser
91
+ test_files: []