blackboard_feed 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +24 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/README.md +22 -0
- data/Rakefile +1 -0
- data/bin/console +14 -0
- data/bin/feed_setup +5 -0
- data/bin/feed_update +5 -0
- data/bin/setup +7 -0
- data/lib/ruby_feed.rb +22 -0
- data/lib/ruby_feed/parser.rb +32 -0
- data/lib/ruby_feed/sql_generator.rb +31 -0
- data/lib/ruby_feed/table_migrator.rb +114 -0
- data/lib/ruby_feed/version.rb +3 -0
- data/ruby_feed.gemspec +24 -0
- metadata +91 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: df77aaf850b61af878f09aee5271db5a6a773e38
|
4
|
+
data.tar.gz: 233190469af03db0a9007a67b81468f4e65b6f96
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b8307ee44f0b0554d175570f4f2a79ba8ef9a5b472d682674bea59972e9f6e8af7a4a5c1cce18261f23b3a05bf9dd990860e267a7ec4efe822554a4d9257f255
|
7
|
+
data.tar.gz: ca4830f8cfc106c9f76fe6b71e15f55980bb7e5d3e8e1335892f1b17292512f92f7ad66159d06af8f85b97789554f6ff59acc851445d5c93e4de786551c2d9f2
|
data/.gitignore
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
*.sql
|
2
|
+
s
|
3
|
+
*.gem
|
4
|
+
*.rbc
|
5
|
+
.bundle
|
6
|
+
.config
|
7
|
+
.yardoc
|
8
|
+
Gemfile.lock
|
9
|
+
InstalledFiles
|
10
|
+
_yardoc
|
11
|
+
coverage
|
12
|
+
doc/
|
13
|
+
lib/bundler/man
|
14
|
+
pkg
|
15
|
+
rdoc
|
16
|
+
spec/reports
|
17
|
+
test/tmp
|
18
|
+
test/version_tmp
|
19
|
+
tmp
|
20
|
+
*.bundle
|
21
|
+
*.so
|
22
|
+
*.o
|
23
|
+
*.a
|
24
|
+
mkmf.log
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# blackboard_feed
|
2
|
+
|
3
|
+
## Setup
|
4
|
+
This program requires a few pieces of software to be installed:
|
5
|
+
* mysql
|
6
|
+
* ruby version >= 1.9.3
|
7
|
+
|
8
|
+
This program requires 3 environment variables to be set:
|
9
|
+
* DATABASE_USER - the user that will log into the mysql database
|
10
|
+
* DATABASE_PASS - the password for that user
|
11
|
+
* FEED_PATH - the absolute path to the blackboard feed files
|
12
|
+
|
13
|
+
To get the databases setup please add the direcotry containing these files to your path and run (TODO):
|
14
|
+
```bash
|
15
|
+
feed_setup.rb
|
16
|
+
```
|
17
|
+
|
18
|
+
## Usage
|
19
|
+
Add the directory that contains these files to your path and run:
|
20
|
+
```bash
|
21
|
+
update.rb
|
22
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "ruby_feed"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/feed_setup
ADDED
data/bin/feed_update
ADDED
data/bin/setup
ADDED
data/lib/ruby_feed.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "ruby_feed/version"
|
2
|
+
require "ruby_feed/sql_generator"
|
3
|
+
require "ruby_feed/table_migrator"
|
4
|
+
require "ruby_feed/parser"
|
5
|
+
|
6
|
+
module RubyFeed
|
7
|
+
class Feed
|
8
|
+
DATABASES = {current: "current", yesterday: "yesterday", two_days_ago: "2_days_ago"}
|
9
|
+
|
10
|
+
def self.setup
|
11
|
+
TableMigrator.create_current(DATABASES[:current])
|
12
|
+
TableMigrator.create_current(DATABASES[:yesterday])
|
13
|
+
TableMigrator.create_current(DATABASES[:two_days_ago])
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.update
|
17
|
+
TableMigrator.drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
|
18
|
+
TableMigrator.drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
|
19
|
+
TableMigrator.create_current(DATABASES[:current])
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
class Parser
|
4
|
+
FEED_FILES_PATH = "/home/jchae/feed/"
|
5
|
+
|
6
|
+
def self.parse_headers(file)
|
7
|
+
raw = %x{ head -1 #{FEED_FILES_PATH}#{file} }
|
8
|
+
parse_header(raw)
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.parse_header(header)
|
12
|
+
headers = header.split("|")
|
13
|
+
headers.map { |h|
|
14
|
+
h.strip!
|
15
|
+
h.chomp!
|
16
|
+
h.downcase!
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.parse_file(lines, db_name)
|
21
|
+
insert_string = ""
|
22
|
+
lines.each do |line|
|
23
|
+
insert_string << "insert into #{db_name} values ("
|
24
|
+
line.split("|").each do |element|
|
25
|
+
element.chomp!
|
26
|
+
insert_string << "'" << element.gsub(/'/,"\\'") << "'" << ","
|
27
|
+
end
|
28
|
+
insert_string << ")"
|
29
|
+
end
|
30
|
+
insert_string.gsub!(/,\)/, "); ")
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
require_relative './parser'
|
3
|
+
|
4
|
+
class SqlGenerator
|
5
|
+
def self.generate_table_create_string(table_name)
|
6
|
+
|
7
|
+
create_string = "create table #{table_name} ("
|
8
|
+
headers = Parser.parse_headers(table_name)
|
9
|
+
|
10
|
+
headers.each { |header|
|
11
|
+
create_string << header << " varchar(255), "
|
12
|
+
}
|
13
|
+
|
14
|
+
create_string.sub!(/, $/, "")
|
15
|
+
create_string << ")"
|
16
|
+
|
17
|
+
create_string
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.generate_drop(db_name)
|
21
|
+
"drop database #{db_name}"
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.generate_rename(to, from)
|
25
|
+
"rename database '#{from}' to '#{to}'"
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.generate_db_create(db_name)
|
29
|
+
"create database #{db_name}"
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,114 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'sequel'
|
4
|
+
require_relative './sql_generator'
|
5
|
+
|
6
|
+
class ::TableMigrator
|
7
|
+
|
8
|
+
FEED_PATH = ENV["FEED_PATH"]
|
9
|
+
DATABASE_USER = ENV["DATABASE_USER"]
|
10
|
+
DATABASE_PASS = ENV["DATABASE_PASS"]
|
11
|
+
|
12
|
+
# def self.setup
|
13
|
+
# create_current(DATABASES[:current])
|
14
|
+
# create_current(DATABASES[:yesterday])
|
15
|
+
# create_current(DATABASES[:two_days_ago])
|
16
|
+
# end
|
17
|
+
|
18
|
+
# def self.update
|
19
|
+
# drop_and_recreate(DATABASES[:two_days_ago], DATABASES[:yesterday])
|
20
|
+
# drop_and_recreate(DATABASES[:yesterday], DATABASES[:current])
|
21
|
+
# create_current(DATABASES[:current])
|
22
|
+
# end
|
23
|
+
|
24
|
+
def self.create_current(db_name)
|
25
|
+
drop(db_name)
|
26
|
+
create(db_name)
|
27
|
+
create_tables(db_name)
|
28
|
+
insert_into(db_name)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.insert_into(db_name)
|
32
|
+
db = connect(db_name)
|
33
|
+
table_names = get_table_names
|
34
|
+
|
35
|
+
table_names.each do |table|
|
36
|
+
table_name = File.basename(table)
|
37
|
+
p "#{FEED_PATH}#{table}"
|
38
|
+
table_file = "#{FEED_PATH}#{table}"
|
39
|
+
insert_string = Parser.parse_file(File.readlines(table_file).drop(1), table_name)
|
40
|
+
next if insert_string.nil?
|
41
|
+
insert_string.split(";").each do |s|
|
42
|
+
next if s == " "
|
43
|
+
# p s
|
44
|
+
db.run(s)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.create_tables(db_name)
|
50
|
+
db = connect(db_name)
|
51
|
+
table_names = get_table_names
|
52
|
+
|
53
|
+
tables = table_names.map {|f| File.basename(f, ".*")}
|
54
|
+
non_existant_tables = tables.reject { |file| db.table_exists?(file) }
|
55
|
+
|
56
|
+
non_existant_tables.each do |table|
|
57
|
+
create_string = SqlGenerator.generate_table_create_string(table)
|
58
|
+
# puts create_string
|
59
|
+
db.run(create_string)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.migrate_info(to, from)
|
64
|
+
drop_and_recreate(to)
|
65
|
+
to = connect(to)
|
66
|
+
from = connect(from)
|
67
|
+
end
|
68
|
+
|
69
|
+
def self.drop_and_recreate(to, from)
|
70
|
+
drop(to)
|
71
|
+
create(to)
|
72
|
+
dump_old_to_new(to, from)
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.dump_old_to_new(to, from)
|
76
|
+
`mysqldump -u #{DATABASE_USER} -p#{DATABASE_PASS} #{from} | mysql -u #{DATABASE_USER} -p#{DATABASE_PASS} #{to}`
|
77
|
+
end
|
78
|
+
|
79
|
+
def self.drop(db_name)
|
80
|
+
db = connect
|
81
|
+
drop_string = SqlGenerator.generate_drop(db_name)
|
82
|
+
db.run(drop_string)
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.rename(to, from)
|
86
|
+
db = connect
|
87
|
+
rename_string = SqlGenerator.generate_rename(to, from)
|
88
|
+
p rename_string
|
89
|
+
db.run(rename_string)
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.create(db_name)
|
93
|
+
db = connect
|
94
|
+
create_string = SqlGenerator.generate_db_create(db_name)
|
95
|
+
db.run(create_string)
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
def self.connect(db_name="mysql")
|
100
|
+
Sequel.connect("mysql2://#{DATABASE_USER}:#{DATABASE_PASS}@localhost/#{db_name}")
|
101
|
+
end
|
102
|
+
|
103
|
+
def self.get_table_names
|
104
|
+
files = Dir.entries(FEED_PATH) - entries_to_be_excluded_from_db
|
105
|
+
files
|
106
|
+
end
|
107
|
+
|
108
|
+
def self.entries_to_be_excluded_from_db
|
109
|
+
[".", "..", "list"]
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
# ::TableMigrator.create_tables("apples")
|
114
|
+
# ::TableMigrator.insert_into("apples")
|
data/ruby_feed.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'ruby_feed/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "blackboard_feed"
|
8
|
+
spec.version = RubyFeed::VERSION
|
9
|
+
spec.authors = ["Jacob Chae"]
|
10
|
+
spec.email = ["jbcden@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = "A feed file parser"
|
13
|
+
spec.description = "This parses feed files for the Oberlin CIT office and loads them into a mysql db"
|
14
|
+
spec.homepage = "https://github.com/jbcden/blackboard_feed"
|
15
|
+
spec.required_ruby_version = ">=1.9.3"
|
16
|
+
spec.license = "MIT"
|
17
|
+
|
18
|
+
spec.files = `git ls-files -z`.split("\x0")
|
19
|
+
spec.executables = ["feed_update", "feed_setup"]
|
20
|
+
spec.require_paths = ["lib"]
|
21
|
+
|
22
|
+
spec.add_development_dependency "bundler", "~> 1.9"
|
23
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: blackboard_feed
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jacob Chae
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-04-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.9'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.9'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: This parses feed files for the Oberlin CIT office and loads them into
|
42
|
+
a mysql db
|
43
|
+
email:
|
44
|
+
- jbcden@gmail.com
|
45
|
+
executables:
|
46
|
+
- feed_update
|
47
|
+
- feed_setup
|
48
|
+
extensions: []
|
49
|
+
extra_rdoc_files: []
|
50
|
+
files:
|
51
|
+
- ".gitignore"
|
52
|
+
- ".travis.yml"
|
53
|
+
- Gemfile
|
54
|
+
- Gemfile.lock
|
55
|
+
- README.md
|
56
|
+
- Rakefile
|
57
|
+
- bin/console
|
58
|
+
- bin/feed_setup
|
59
|
+
- bin/feed_update
|
60
|
+
- bin/setup
|
61
|
+
- lib/ruby_feed.rb
|
62
|
+
- lib/ruby_feed/parser.rb
|
63
|
+
- lib/ruby_feed/sql_generator.rb
|
64
|
+
- lib/ruby_feed/table_migrator.rb
|
65
|
+
- lib/ruby_feed/version.rb
|
66
|
+
- ruby_feed.gemspec
|
67
|
+
homepage: https://github.com/jbcden/blackboard_feed
|
68
|
+
licenses:
|
69
|
+
- MIT
|
70
|
+
metadata: {}
|
71
|
+
post_install_message:
|
72
|
+
rdoc_options: []
|
73
|
+
require_paths:
|
74
|
+
- lib
|
75
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: 1.9.3
|
80
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ">="
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
requirements: []
|
86
|
+
rubyforge_project:
|
87
|
+
rubygems_version: 2.4.6
|
88
|
+
signing_key:
|
89
|
+
specification_version: 4
|
90
|
+
summary: A feed file parser
|
91
|
+
test_files: []
|