gtfs-meta 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +163 -0
- data/Rakefile +1 -0
- data/gtfs-meta.gemspec +25 -0
- data/lib/gtfs.rb +4 -0
- data/lib/gtfs/meta.rb +22 -0
- data/lib/gtfs/meta/generators/.gitkeep +0 -0
- data/lib/gtfs/meta/migrations/.gitkeep +0 -0
- data/lib/gtfs/meta/migrations/create_feed_version_checks.rb +17 -0
- data/lib/gtfs/meta/migrations/create_feed_versions.rb +20 -0
- data/lib/gtfs/meta/migrations/create_feeds.rb +20 -0
- data/lib/gtfs/meta/migrations/create_publishers.rb +17 -0
- data/lib/gtfs/meta/models/feed.rb +27 -0
- data/lib/gtfs/meta/models/feed_version.rb +15 -0
- data/lib/gtfs/meta/models/feed_version_check.rb +7 -0
- data/lib/gtfs/meta/models/publisher.rb +7 -0
- data/lib/gtfs/meta/version.rb +5 -0
- data/lib/gtfs/meta/workers/feed_manager.rb +96 -0
- data/lib/gtfs/meta/workers/feed_seeder.rb +13 -0
- metadata +132 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 MJ Rossetti
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
# GTFS::Meta
|
2
|
+
|
3
|
+
A [General Transit Feed Specification (GTFS)](https://developers.google.com/transit/gtfs/) data and metadata manager for ActiveRecord.
|
4
|
+
|
5
|
+
Extracts feed data, manages feed versions, and extends the feed specification to include feed metadata.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'gtfs-meta'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install gtfs-meta
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
### Migrating the Database
|
26
|
+
|
27
|
+
```` rb
|
28
|
+
GTFS::Meta::CreatePublishers.migrate(:up)
|
29
|
+
GTFS::Meta::CreateFeeds.migrate(:up)
|
30
|
+
GTFS::Meta::CreateFeedVersions.migrate(:up)
|
31
|
+
GTFS::Meta::CreateFeedVersionChecks.migrate(:up)
|
32
|
+
````
|
33
|
+
|
34
|
+
### Feed-Seeding
|
35
|
+
|
36
|
+
Pass custom seed data to the [FeedSeeder](lib/gtfs/meta/workers/feed_seeder.rb) to persist valid feeds and publishers to the database.
|
37
|
+
|
38
|
+
```` rb
|
39
|
+
MY_PUBLISHERS = [
|
40
|
+
{:id => 1, :name => "Shore Line East", :url => "http://www.shorelineeast.com", :feeds_url => nil, :email_address => nil},
|
41
|
+
{:id => 2, :name => "Metropolitan Transportation Authority", :url => "http://web.mta.info/developers", :feeds_url => "http://web.mta.info/developers/developer-data-terms.html#data", :email_address => nil},
|
42
|
+
{:id => 3, :name => "CT Transit", :url => "http://www.cttransit.com/", :feeds_url => "http://www.cttransit.com/about/developers/gtfsdata/Main.asp", :email_address => nil}
|
43
|
+
]
|
44
|
+
|
45
|
+
MY_FEEDS = [
|
46
|
+
{:publisher_id => 1, :source_url => "http://www.shorelineeast.com/google_transit.zip", :source_title => "Shore Line East"},
|
47
|
+
{:publisher_id => 2, :source_url => "http://web.mta.info/developers/data/mnr/google_transit.zip", :source_title => "Metro-North Railroad"},
|
48
|
+
{:publisher_id => 2, :source_url => "http://web.mta.info/developers/data/nyct/subway/google_transit.zip", :source_title => "New York City Transit Subway"},
|
49
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googleha_transit.zip", :source_title => "Hartford"},
|
50
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlenh_transit.zip", :source_title => "New Haven"},
|
51
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlestam_transit.zip", :source_title => "Stamford"},
|
52
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlewat_transit.zip", :source_title => "Waterbury"},
|
53
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlenb_transit.zip", :source_title => "New Britain"},
|
54
|
+
{:publisher_id => 3,:source_url => "http://www.cttransit.com/uploads_GTFS/googleme_transit.zip",:source_title => "Meriden"}
|
55
|
+
]
|
56
|
+
|
57
|
+
GTFS::Meta::FeedSeeder.perform(:publishers => MY_PUBLISHERS, :feeds => MY_FEEDS)
|
58
|
+
````
|
59
|
+
|
60
|
+
Consult the [reference documentation](README.md#reference) for guidance on defining attributes for feeds and associated publishers.
|
61
|
+
|
62
|
+
### Feed-Managing
|
63
|
+
|
64
|
+
Once persisted, pass one or more feeds to the [FeedManager](lib/gtfs/meta/workers/feed_manager.rb).
|
65
|
+
|
66
|
+
```` rb
|
67
|
+
feeds = GTFS::Meta::Feed.all
|
68
|
+
GTFS::Meta::FeedManager.perform(:feeds => feeds)
|
69
|
+
````
|
70
|
+
|
71
|
+
This script checks for new feed versions, downloads feed data, and stores it according to the following directory structure: `db/gtfs/publishers/PUBLISHER_NAME/feeds/FEED_NAME/versions/ETAG`
|
72
|
+
````
|
73
|
+
db/gtfs/publishers/cttransit/feeds/newhaven/versions/d6594392f3c6cf11254/
|
74
|
+
+ agency.txt
|
75
|
+
+ calendar_dates.txt
|
76
|
+
+ routes.txt
|
77
|
+
+ shapes.txt
|
78
|
+
+ stop_times.txt
|
79
|
+
+ stops.txt
|
80
|
+
+ trips.txt
|
81
|
+
|
82
|
+
db/gtfs/publishers/shorelineeast/feeds/shorelineeast/versions/26349c817da114d66c0/
|
83
|
+
+ agency.txt
|
84
|
+
+ calendar_dates.txt
|
85
|
+
+ fare_attributes.txt
|
86
|
+
+ fare_rules.txt
|
87
|
+
+ routes.txt
|
88
|
+
+ stop_times.txt
|
89
|
+
+ stops.txt
|
90
|
+
+ trips.txt
|
91
|
+
````
|
92
|
+
|
93
|
+
## Reference
|
94
|
+
|
95
|
+
### Publisher
|
96
|
+
|
97
|
+
A `Publisher` is a type of [`Agency`](https://developers.google.com/transit/gtfs/reference#agency_fields) that provides authoritative source data for a given `Feed`. A publisher may host one or more feeds.
|
98
|
+
|
99
|
+
#### Field Definitions:
|
100
|
+
|
101
|
+
#### Sample Data:
|
102
|
+
|
103
|
+
id | name | url | feeds_url | email_address | created_at | updated_at
|
104
|
+
--- | --- | --- | --- | --- | --- | ---
|
105
|
+
1 | Shore Line East | http://www.shorelineeast.com | NULL | NULL | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
106
|
+
2 | Metropolitan Transportation Authority | http://web.mta.info/developers | http://web.mta.info/developers/developer-data-terms.html#data | NULL | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
107
|
+
3 | CT Transit | http://www.cttransit.com/ | http://www.cttransit.com/about/developers/gtfsdata/Main.asp | NULL 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
108
|
+
|
109
|
+
### Feed
|
110
|
+
|
111
|
+
A `Feed` represents a dedicated source of GTFS data available for download in `.zip` format over `http`. Feeds are published and modified by their respective publishers.
|
112
|
+
|
113
|
+
#### Field Definitions:
|
114
|
+
|
115
|
+
#### Sample Data:
|
116
|
+
|
117
|
+
id | publisher_id | source_url | source_title | created_at | updated_at
|
118
|
+
--- | --- | --- | --- | --- | ---
|
119
|
+
1 | 1 | http://www.shorelineeast.com/google_transit.zip | Shore Line East | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
120
|
+
2 | 2 | http://web.mta.info/developers/data/mnr/google_transit.zip | Metro-North Railroad | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
121
|
+
3 | 2 | http://web.mta.info/developers/data/nyct/subway/google_transit.zip | New York City Transit Subway | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
122
|
+
4 | 3 | http://www.cttransit.com/uploads_GTFS/googleha_transit.zip | Hartford | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
123
|
+
5 | 3 | http://www.cttransit.com/uploads_GTFS/googlenh_transit.zip | New Haven | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
124
|
+
|
125
|
+
> NOTE: a feed's source attributes (`source_url` and `source_title`) originate from the feed's html link attributes: `<a href="SOURCE_URL">SOURCE_TITLE</a>`.
|
126
|
+
|
127
|
+
### FeedVersion
|
128
|
+
|
129
|
+
A `FeedVersion` is a collective modification of feed files provided by a feed publisher. New feed versions supercede previous versions, and are effective for a limited period of time.
|
130
|
+
|
131
|
+
#### Field Definitions:
|
132
|
+
|
133
|
+
#### Sample Data:
|
134
|
+
|
135
|
+
id | feed_id | etag | last_modified_at | file_names | created_at | updated_at
|
136
|
+
--- | --- |--- | --- | --- | --- | ---
|
137
|
+
1 | 1 | 26349c817da114d66c0 | 2014-10-03 13:54:11 |some binary output | 2014-11-06 12:43:08 | 2014-11-09 18:50:55
|
138
|
+
2 | 2 | W683aa544eaa01 | 2014-10-27 20:24:33 |some binary output | 2014-11-06 12:43:10 | 2014-11-09 18:50:57
|
139
|
+
3 | 3 | W4cd511541c8db0 | 2014-09-19 20:10:24 |some binary output | 2014-11-06 12:43:26 | 2014-11-09 18:51:08
|
140
|
+
4 | 4 | 329c33d849d4cf1124e | 2014-09-19 20:39:44 |some binary output | 2014-11-06 12:43:38 | 2014-11-06 12:46:47
|
141
|
+
5 | 5 | d6594392f3c6cf1124e | 2014-09-02 21:19:25 |some binary output | 2014-11-06 12:43:47 | 2014-11-06 12:46:56
|
142
|
+
6 | 6 | 8db438eb8eb8cf1124e | 2014-08-15 13:43:38 |some binary output | 2014-11-06 12:43:51 | 2014-11-06 12:46:59
|
143
|
+
7 | 7 | 28f16c8c481cf1124e | 2014-06-06 20:20:38 |some binary output | 2014-11-06 12:43:53 | 2014-11-06 12:47:00
|
144
|
+
8 | 8 | faac30c98aadcf1124e | 2014-08-01 13:16:20 |some binary output | 2014-11-06 12:43:55 | 2014-11-06 12:47:02
|
145
|
+
9 | 9 | b4bd11ecc481cf1124e | 2014-06-06 20:21:39 |some binary output | 2014-11-06 12:43:55 | 2014-11-06 12:47:03
|
146
|
+
10 | 4 | 329c33d849d4cf11254 | 2014-09-19 20:39:44 |some binary output | 2014-11-08 06:37:38 | 2014-11-09 18:51:22
|
147
|
+
11 | 5 | d6594392f3c6cf11254 | 2014-09-02 21:19:25 |some binary output | 2014-11-08 06:37:50 | 2014-11-09 18:51:31
|
148
|
+
12 | 6 | 8db438eb8eb8cf11254 | 2014-08-15 13:43:38 |some binary output | 2014-11-08 06:37:54 | 2014-11-09 18:51:33
|
149
|
+
13 | 7 | 28f16c8c481cf11254 | 2014-06-06 20:20:38 |some binary output | 2014-11-08 06:37:56 | 2014-11-09 18:51:36
|
150
|
+
14 | 8 | faac30c98aadcf11254 | 2014-08-01 13:16:20 |some binary output | 2014-11-08 06:37:59 | 2014-11-09 18:51:37
|
151
|
+
15 | 9 | b4bd11ecc481cf11254 | 2014-06-06 20:21:39 |some binary output | 2014-11-08 06:38:00 | 2014-11-09 18:51:38
|
152
|
+
|
153
|
+
### FeedFile
|
154
|
+
|
155
|
+
A [`FeedFile`](https://developers.google.com/transit/gtfs/reference#FeedFiles) is a physical `.txt` file of GTFS data containing one or more versioned instances of a given [`GTFS::Model`](https://github.com/nerdEd/gtfs/blob/master/lib/gtfs/model.rb). Feed files are released collectively in versions by their respective publisher.
|
156
|
+
|
157
|
+
## Contributing
|
158
|
+
|
159
|
+
1. Fork it ( https://github.com/databyday/gtfs-meta/fork )
|
160
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
161
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
162
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
163
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/gtfs-meta.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'gtfs/meta/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "gtfs-meta"
|
8
|
+
spec.version = GTFS::Meta::VERSION
|
9
|
+
spec.authors = ["MJ Rossetti"]
|
10
|
+
spec.email = ["s2t2mail@gmail.com"]
|
11
|
+
spec.summary = %q{A GTFS data and metadata manager for ActiveRecord.}
|
12
|
+
spec.description = %q{Extracts feed data, manages feed versions, and extends the feed specification to include feed metadata.}
|
13
|
+
spec.homepage = "https://github.com/databyday/gtfs-meta"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_dependency "activerecord", "~> 4.0.2"
|
24
|
+
spec.add_dependency "gtfs", "~> 0.2.4"
|
25
|
+
end
|
data/lib/gtfs.rb
ADDED
data/lib/gtfs/meta.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "gtfs"
|
2
|
+
|
3
|
+
module GTFS
|
4
|
+
module Meta
|
5
|
+
MESSAGE = "open transit data ftw"
|
6
|
+
GTFS_DATA_DIRECTORY = "db/gtfs" #todo: allow configuration
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
require "gtfs/meta/version"
|
11
|
+
|
12
|
+
require "gtfs/meta/migrations/create_publishers"
|
13
|
+
require "gtfs/meta/migrations/create_feeds"
|
14
|
+
require "gtfs/meta/models/publisher"
|
15
|
+
require "gtfs/meta/models/feed"
|
16
|
+
require "gtfs/meta/workers/feed_seeder"
|
17
|
+
|
18
|
+
require "gtfs/meta/migrations/create_feed_versions"
|
19
|
+
require "gtfs/meta/migrations/create_feed_version_checks"
|
20
|
+
require "gtfs/meta/models/feed_version"
|
21
|
+
require "gtfs/meta/models/feed_version_check"
|
22
|
+
require "gtfs/meta/workers/feed_manager"
|
File without changes
|
File without changes
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeedVersionChecks < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feed_version_checks do |t|
|
8
|
+
t.integer :feed_version_id, :null => false
|
9
|
+
t.string :status
|
10
|
+
t.timestamps
|
11
|
+
end
|
12
|
+
|
13
|
+
add_index :feed_version_checks, :feed_version_id
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeedVersions < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feed_versions do |t|
|
8
|
+
t.integer :feed_id, :null => false
|
9
|
+
t.string :etag, :null => false
|
10
|
+
t.datetime :last_modified_at, :null => false
|
11
|
+
t.text :file_names
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
|
15
|
+
add_index :feed_versions, :feed_id
|
16
|
+
add_index :feed_versions, [:feed_id, :etag, :last_modified_at], :unique => true, :name => "version_composite_key"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeeds < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feeds do |t|
|
8
|
+
t.integer :publisher_id, :null => false
|
9
|
+
t.string :source_url, :null => false
|
10
|
+
t.string :source_title, :null => false
|
11
|
+
t.timestamps
|
12
|
+
end
|
13
|
+
|
14
|
+
add_index :feeds, :publisher_id
|
15
|
+
add_index :feeds, :source_url, :unique => true
|
16
|
+
add_index :feeds, [:publisher_id, :source_title], :unique => true, :name => "publisher_unique_source_titles"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreatePublishers < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :publishers do |t|
|
8
|
+
t.string :name
|
9
|
+
t.string :url, :null => false
|
10
|
+
t.string :feeds_url
|
11
|
+
t.string :email_address
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module GTFS
|
4
|
+
module Meta
|
5
|
+
class Feed < ActiveRecord::Base
|
6
|
+
#TODO: re-construct this model from a custom generator
|
7
|
+
|
8
|
+
belongs_to :publisher, :inverse_of => :feeds, :class_name => "GTFS::Meta::Publisher"
|
9
|
+
|
10
|
+
has_many :versions, :inverse_of => :feed, :class_name => "GTFS::Meta::FeedVersion"
|
11
|
+
|
12
|
+
validate :source_zipness_validator
|
13
|
+
|
14
|
+
def source_zipness_validator
|
15
|
+
errors.add(:source_url, "is not a zip") unless (source_url && source_file_name.ends_with?(".zip"))
|
16
|
+
end
|
17
|
+
|
18
|
+
def source_host_name
|
19
|
+
URI(source_url).hostname.split(".").reverse[1]
|
20
|
+
end
|
21
|
+
|
22
|
+
def source_file_name
|
23
|
+
source_url.split("/").last
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedVersion < ActiveRecord::Base
|
4
|
+
#TODO: re-construct this model from a custom generator
|
5
|
+
|
6
|
+
belongs_to :feed, :inverse_of => :versions, :class_name => "GTFS::Meta::Feed"
|
7
|
+
|
8
|
+
serialize :file_names, Array
|
9
|
+
|
10
|
+
def destination_path
|
11
|
+
"#{GTFS_DATA_DIRECTORY}/publishers/#{feed.source_host_name}/feeds/#{feed.source_title.downcase.gsub("-","").gsub(" ","")}/versions/#{etag}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedManager
|
4
|
+
FEED_FILE_NAMES = GTFS::Source::ENTITIES.map{|e| e.name == "Agency" ? e.singular_name.to_s.concat(".txt") : e.name.to_s.concat(".txt")}
|
5
|
+
|
6
|
+
def self.perform(options = {})
|
7
|
+
feeds = options[:feeds] || GTFS::Meta::Feed.all
|
8
|
+
|
9
|
+
feeds.find_each do |feed|
|
10
|
+
begin
|
11
|
+
# CHECK VERSION
|
12
|
+
|
13
|
+
uri = URI.parse(feed.source_url)
|
14
|
+
response = Net::HTTP.get_response(uri)
|
15
|
+
etag = response.to_hash["etag"].first.gsub!(/[^0-9A-Za-z]/, '')
|
16
|
+
last_modified_at = response.to_hash["last-modified"].first.try(:to_datetime)
|
17
|
+
|
18
|
+
raise EtagError.new(etag) unless etag.is_a?(String)
|
19
|
+
raise LastModifiedError.new(last_modified_at) unless last_modified_at.is_a?(DateTime)
|
20
|
+
feed_version = GTFS::Meta::FeedVersion.where({
|
21
|
+
:feed_id => feed.id,
|
22
|
+
:etag => etag,
|
23
|
+
:last_modified_at => last_modified_at
|
24
|
+
}).first_or_create
|
25
|
+
feed_version_check = GTFS::Meta::FeedVersionCheck.create({
|
26
|
+
:feed_version_id => feed_version.id,
|
27
|
+
:status => "IN PROGRESS"
|
28
|
+
})
|
29
|
+
|
30
|
+
# MANAGE FILES
|
31
|
+
|
32
|
+
FileUtils.mkdir_p(feed_version.destination_path)
|
33
|
+
|
34
|
+
zip_destination_path = "#{feed_version.destination_path}/#{feed.source_file_name}"
|
35
|
+
unless File.exist?(zip_destination_path)
|
36
|
+
File.open(zip_destination_path, "wb") do |zip_file|
|
37
|
+
zip_file.write response.body
|
38
|
+
end
|
39
|
+
raise SourceExtractionError.new(zip_destination_path) unless File.exist?(zip_destination_path)
|
40
|
+
end
|
41
|
+
|
42
|
+
Zip::File.open(zip_destination_path) do |zip_file|
|
43
|
+
zip_file.each do |entry|
|
44
|
+
begin
|
45
|
+
entry_name = entry.name
|
46
|
+
raise InvalidEntryName.new(entry_name) unless FEED_FILE_NAMES.include?(entry_name)
|
47
|
+
feed_version.file_names << entry_name unless feed_version.file_names.include?(entry_name)
|
48
|
+
|
49
|
+
feed_file_path = "#{feed_version.destination_path}/#{entry_name}"
|
50
|
+
raise FeedFileExists.new(feed_file_path) if File.exist?(feed_file_path)
|
51
|
+
|
52
|
+
entry.extract(feed_file_path)
|
53
|
+
raise FeedFileExtractionError.new(feed_file_path) unless File.exist?(feed_file_path)
|
54
|
+
rescue InvalidEntryName => e
|
55
|
+
next
|
56
|
+
rescue FeedFileExists => e
|
57
|
+
next
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
FileUtils.rm(zip_destination_path)
|
63
|
+
raise SourceRemovalError.new(zip_destination_path) if File.exist?(zip_destination_path)
|
64
|
+
|
65
|
+
feed_version.save!
|
66
|
+
feed_version_check.update_attributes(:status => "SUCCESS")
|
67
|
+
rescue => e
|
68
|
+
puts "#{e.class} -- #{e.message}"
|
69
|
+
feed_version_check.update_attributes(:status => "FAILURE") if feed_version_check
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class EtagError < StandardError
|
75
|
+
end
|
76
|
+
|
77
|
+
class LastModifiedError < StandardError
|
78
|
+
end
|
79
|
+
|
80
|
+
class SourceExtractionError < StandardError
|
81
|
+
end
|
82
|
+
|
83
|
+
class InvalidEntryName < StandardError
|
84
|
+
end
|
85
|
+
|
86
|
+
class FeedFileExists < StandardError
|
87
|
+
end
|
88
|
+
|
89
|
+
class FeedFileExtractionError < StandardError
|
90
|
+
end
|
91
|
+
|
92
|
+
class SourceRemovalError < StandardError
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedSeeder
|
4
|
+
def self.perform(options = {})
|
5
|
+
publishers = options[:publishers]
|
6
|
+
feeds = options[:feeds]
|
7
|
+
|
8
|
+
GTFS::Meta::Publisher.first_or_create(publishers)
|
9
|
+
GTFS::Meta::Feed.first_or_create(feeds)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gtfs-meta
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- MJ Rossetti
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-11-23 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.7'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.7'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '10.0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '10.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: activerecord
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 4.0.2
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 4.0.2
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: gtfs
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.2.4
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.2.4
|
78
|
+
description: Extracts feed data, manages feed versions, and extends the feed specification
|
79
|
+
to include feed metadata.
|
80
|
+
email:
|
81
|
+
- s2t2mail@gmail.com
|
82
|
+
executables: []
|
83
|
+
extensions: []
|
84
|
+
extra_rdoc_files: []
|
85
|
+
files:
|
86
|
+
- .gitignore
|
87
|
+
- Gemfile
|
88
|
+
- LICENSE.txt
|
89
|
+
- README.md
|
90
|
+
- Rakefile
|
91
|
+
- gtfs-meta.gemspec
|
92
|
+
- lib/gtfs.rb
|
93
|
+
- lib/gtfs/meta.rb
|
94
|
+
- lib/gtfs/meta/generators/.gitkeep
|
95
|
+
- lib/gtfs/meta/migrations/.gitkeep
|
96
|
+
- lib/gtfs/meta/migrations/create_feed_version_checks.rb
|
97
|
+
- lib/gtfs/meta/migrations/create_feed_versions.rb
|
98
|
+
- lib/gtfs/meta/migrations/create_feeds.rb
|
99
|
+
- lib/gtfs/meta/migrations/create_publishers.rb
|
100
|
+
- lib/gtfs/meta/models/feed.rb
|
101
|
+
- lib/gtfs/meta/models/feed_version.rb
|
102
|
+
- lib/gtfs/meta/models/feed_version_check.rb
|
103
|
+
- lib/gtfs/meta/models/publisher.rb
|
104
|
+
- lib/gtfs/meta/version.rb
|
105
|
+
- lib/gtfs/meta/workers/feed_manager.rb
|
106
|
+
- lib/gtfs/meta/workers/feed_seeder.rb
|
107
|
+
homepage: https://github.com/databyday/gtfs-meta
|
108
|
+
licenses:
|
109
|
+
- MIT
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
none: false
|
116
|
+
requirements:
|
117
|
+
- - ! '>='
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
requirements: []
|
127
|
+
rubyforge_project:
|
128
|
+
rubygems_version: 1.8.23
|
129
|
+
signing_key:
|
130
|
+
specification_version: 3
|
131
|
+
summary: A GTFS data and metadata manager for ActiveRecord.
|
132
|
+
test_files: []
|