gtfs-meta 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +14 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +163 -0
- data/Rakefile +1 -0
- data/gtfs-meta.gemspec +25 -0
- data/lib/gtfs.rb +4 -0
- data/lib/gtfs/meta.rb +22 -0
- data/lib/gtfs/meta/generators/.gitkeep +0 -0
- data/lib/gtfs/meta/migrations/.gitkeep +0 -0
- data/lib/gtfs/meta/migrations/create_feed_version_checks.rb +17 -0
- data/lib/gtfs/meta/migrations/create_feed_versions.rb +20 -0
- data/lib/gtfs/meta/migrations/create_feeds.rb +20 -0
- data/lib/gtfs/meta/migrations/create_publishers.rb +17 -0
- data/lib/gtfs/meta/models/feed.rb +27 -0
- data/lib/gtfs/meta/models/feed_version.rb +15 -0
- data/lib/gtfs/meta/models/feed_version_check.rb +7 -0
- data/lib/gtfs/meta/models/publisher.rb +7 -0
- data/lib/gtfs/meta/version.rb +5 -0
- data/lib/gtfs/meta/workers/feed_manager.rb +96 -0
- data/lib/gtfs/meta/workers/feed_seeder.rb +13 -0
- metadata +132 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 MJ Rossetti
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,163 @@
|
|
1
|
+
# GTFS::Meta
|
2
|
+
|
3
|
+
A [General Transit Feed Specification (GTFS)](https://developers.google.com/transit/gtfs/) data and metadata manager for ActiveRecord.
|
4
|
+
|
5
|
+
Extracts feed data, manages feed versions, and extends the feed specification to include feed metadata.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'gtfs-meta'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install gtfs-meta
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
### Migrating the Database
|
26
|
+
|
27
|
+
```` rb
|
28
|
+
GTFS::Meta::CreatePublishers.migrate(:up)
|
29
|
+
GTFS::Meta::CreateFeeds.migrate(:up)
|
30
|
+
GTFS::Meta::CreateFeedVersions.migrate(:up)
|
31
|
+
GTFS::Meta::CreateFeedVersionChecks.migrate(:up)
|
32
|
+
````
|
33
|
+
|
34
|
+
### Feed-Seeding
|
35
|
+
|
36
|
+
Pass custom seed data to the [FeedSeeder](lib/gtfs/meta/workers/feed_seeder.rb) to persist valid feeds and publishers to the database.
|
37
|
+
|
38
|
+
```` rb
|
39
|
+
MY_PUBLISHERS = [
|
40
|
+
{:id => 1, :name => "Shore Line East", :url => "http://www.shorelineeast.com", :feeds_url => nil, :email_address => nil},
|
41
|
+
{:id => 2, :name => "Metropolitan Transportation Authority", :url => "http://web.mta.info/developers", :feeds_url => "http://web.mta.info/developers/developer-data-terms.html#data", :email_address => nil},
|
42
|
+
{:id => 3, :name => "CT Transit", :url => "http://www.cttransit.com/", :feeds_url => "http://www.cttransit.com/about/developers/gtfsdata/Main.asp", :email_address => nil}
|
43
|
+
]
|
44
|
+
|
45
|
+
MY_FEEDS = [
|
46
|
+
{:publisher_id => 1, :source_url => "http://www.shorelineeast.com/google_transit.zip", :source_title => "Shore Line East"},
|
47
|
+
{:publisher_id => 2, :source_url => "http://web.mta.info/developers/data/mnr/google_transit.zip", :source_title => "Metro-North Railroad"},
|
48
|
+
{:publisher_id => 2, :source_url => "http://web.mta.info/developers/data/nyct/subway/google_transit.zip", :source_title => "New York City Transit Subway"},
|
49
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googleha_transit.zip", :source_title => "Hartford"},
|
50
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlenh_transit.zip", :source_title => "New Haven"},
|
51
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlestam_transit.zip", :source_title => "Stamford"},
|
52
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlewat_transit.zip", :source_title => "Waterbury"},
|
53
|
+
{:publisher_id => 3, :source_url => "http://www.cttransit.com/uploads_GTFS/googlenb_transit.zip", :source_title => "New Britain"},
|
54
|
+
{:publisher_id => 3,:source_url => "http://www.cttransit.com/uploads_GTFS/googleme_transit.zip",:source_title => "Meriden"}
|
55
|
+
]
|
56
|
+
|
57
|
+
GTFS::Meta::FeedSeeder.perform(:publishers => MY_PUBLISHERS, :feeds => MY_FEEDS)
|
58
|
+
````
|
59
|
+
|
60
|
+
Consult the [reference documentation](README.md#reference) for guidance on defining attributes for feeds and associated publishers.
|
61
|
+
|
62
|
+
### Feed-Managing
|
63
|
+
|
64
|
+
Once persisted, pass one or more feeds to the [FeedManager](lib/gtfs/meta/workers/feed_manager.rb).
|
65
|
+
|
66
|
+
```` rb
|
67
|
+
feeds = GTFS::Meta::Feed.all
|
68
|
+
GTFS::Meta::FeedManager.perform(:feeds => feeds)
|
69
|
+
````
|
70
|
+
|
71
|
+
This script checks for new feed versions, downloads feed data, and stores it according to the following directory structure: `db/gtfs/publishers/PUBLISHER_NAME/feeds/FEED_NAME/versions/ETAG`
|
72
|
+
````
|
73
|
+
db/gtfs/publishers/cttransit/feeds/newhaven/versions/d6594392f3c6cf11254/
|
74
|
+
+ agency.txt
|
75
|
+
+ calendar_dates.txt
|
76
|
+
+ routes.txt
|
77
|
+
+ shapes.txt
|
78
|
+
+ stop_times.txt
|
79
|
+
+ stops.txt
|
80
|
+
+ trips.txt
|
81
|
+
|
82
|
+
db/gtfs/publishers/shorelineeast/feeds/shorelineeast/versions/26349c817da114d66c0/
|
83
|
+
+ agency.txt
|
84
|
+
+ calendar_dates.txt
|
85
|
+
+ fare_attributes.txt
|
86
|
+
+ fare_rules.txt
|
87
|
+
+ routes.txt
|
88
|
+
+ stop_times.txt
|
89
|
+
+ stops.txt
|
90
|
+
+ trips.txt
|
91
|
+
````
|
92
|
+
|
93
|
+
## Reference
|
94
|
+
|
95
|
+
### Publisher
|
96
|
+
|
97
|
+
A `Publisher` is a type of [`Agency`](https://developers.google.com/transit/gtfs/reference#agency_fields) that provides authoritative source data for a given `Feed`. A publisher may host one or more feeds.
|
98
|
+
|
99
|
+
#### Field Definitions:
|
100
|
+
|
101
|
+
#### Sample Data:
|
102
|
+
|
103
|
+
id | name | url | feeds_url | email_address | created_at | updated_at
|
104
|
+
--- | --- | --- | --- | --- | --- | ---
|
105
|
+
1 | Shore Line East | http://www.shorelineeast.com | NULL | NULL | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
106
|
+
2 | Metropolitan Transportation Authority | http://web.mta.info/developers | http://web.mta.info/developers/developer-data-terms.html#data | NULL | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
107
|
+
3 | CT Transit | http://www.cttransit.com/ | http://www.cttransit.com/about/developers/gtfsdata/Main.asp | NULL 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
108
|
+
|
109
|
+
### Feed
|
110
|
+
|
111
|
+
A `Feed` represents a dedicated source of GTFS data available for download in `.zip` format over `http`. Feeds are published and modified by their respective publishers.
|
112
|
+
|
113
|
+
#### Field Definitions:
|
114
|
+
|
115
|
+
#### Sample Data:
|
116
|
+
|
117
|
+
id | publisher_id | source_url | source_title | created_at | updated_at
|
118
|
+
--- | --- | --- | --- | --- | ---
|
119
|
+
1 | 1 | http://www.shorelineeast.com/google_transit.zip | Shore Line East | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
120
|
+
2 | 2 | http://web.mta.info/developers/data/mnr/google_transit.zip | Metro-North Railroad | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
121
|
+
3 | 2 | http://web.mta.info/developers/data/nyct/subway/google_transit.zip | New York City Transit Subway | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
122
|
+
4 | 3 | http://www.cttransit.com/uploads_GTFS/googleha_transit.zip | Hartford | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
123
|
+
5 | 3 | http://www.cttransit.com/uploads_GTFS/googlenh_transit.zip | New Haven | 2014-11-06 12:42:52 | 2014-11-06 12:42:52
|
124
|
+
|
125
|
+
> NOTE: a feed's source attributes (`source_url` and `source_title`) originate from the feed's html link attributes: `<a href="SOURCE_URL">SOURCE_TITLE</a>`.
|
126
|
+
|
127
|
+
### FeedVersion
|
128
|
+
|
129
|
+
A `FeedVersion` is a collective modification of feed files provided by a feed publisher. New feed versions supercede previous versions, and are effective for a limited period of time.
|
130
|
+
|
131
|
+
#### Field Definitions:
|
132
|
+
|
133
|
+
#### Sample Data:
|
134
|
+
|
135
|
+
id | feed_id | etag | last_modified_at | file_names | created_at | updated_at
|
136
|
+
--- | --- |--- | --- | --- | --- | ---
|
137
|
+
1 | 1 | 26349c817da114d66c0 | 2014-10-03 13:54:11 |some binary output | 2014-11-06 12:43:08 | 2014-11-09 18:50:55
|
138
|
+
2 | 2 | W683aa544eaa01 | 2014-10-27 20:24:33 |some binary output | 2014-11-06 12:43:10 | 2014-11-09 18:50:57
|
139
|
+
3 | 3 | W4cd511541c8db0 | 2014-09-19 20:10:24 |some binary output | 2014-11-06 12:43:26 | 2014-11-09 18:51:08
|
140
|
+
4 | 4 | 329c33d849d4cf1124e | 2014-09-19 20:39:44 |some binary output | 2014-11-06 12:43:38 | 2014-11-06 12:46:47
|
141
|
+
5 | 5 | d6594392f3c6cf1124e | 2014-09-02 21:19:25 |some binary output | 2014-11-06 12:43:47 | 2014-11-06 12:46:56
|
142
|
+
6 | 6 | 8db438eb8eb8cf1124e | 2014-08-15 13:43:38 |some binary output | 2014-11-06 12:43:51 | 2014-11-06 12:46:59
|
143
|
+
7 | 7 | 28f16c8c481cf1124e | 2014-06-06 20:20:38 |some binary output | 2014-11-06 12:43:53 | 2014-11-06 12:47:00
|
144
|
+
8 | 8 | faac30c98aadcf1124e | 2014-08-01 13:16:20 |some binary output | 2014-11-06 12:43:55 | 2014-11-06 12:47:02
|
145
|
+
9 | 9 | b4bd11ecc481cf1124e | 2014-06-06 20:21:39 |some binary output | 2014-11-06 12:43:55 | 2014-11-06 12:47:03
|
146
|
+
10 | 4 | 329c33d849d4cf11254 | 2014-09-19 20:39:44 |some binary output | 2014-11-08 06:37:38 | 2014-11-09 18:51:22
|
147
|
+
11 | 5 | d6594392f3c6cf11254 | 2014-09-02 21:19:25 |some binary output | 2014-11-08 06:37:50 | 2014-11-09 18:51:31
|
148
|
+
12 | 6 | 8db438eb8eb8cf11254 | 2014-08-15 13:43:38 |some binary output | 2014-11-08 06:37:54 | 2014-11-09 18:51:33
|
149
|
+
13 | 7 | 28f16c8c481cf11254 | 2014-06-06 20:20:38 |some binary output | 2014-11-08 06:37:56 | 2014-11-09 18:51:36
|
150
|
+
14 | 8 | faac30c98aadcf11254 | 2014-08-01 13:16:20 |some binary output | 2014-11-08 06:37:59 | 2014-11-09 18:51:37
|
151
|
+
15 | 9 | b4bd11ecc481cf11254 | 2014-06-06 20:21:39 |some binary output | 2014-11-08 06:38:00 | 2014-11-09 18:51:38
|
152
|
+
|
153
|
+
### FeedFile
|
154
|
+
|
155
|
+
A [`FeedFile`](https://developers.google.com/transit/gtfs/reference#FeedFiles) is a physical `.txt` file of GTFS data containing one or more versioned instances of a given [`GTFS::Model`](https://github.com/nerdEd/gtfs/blob/master/lib/gtfs/model.rb). Feed files are released collectively in versions by their respective publisher.
|
156
|
+
|
157
|
+
## Contributing
|
158
|
+
|
159
|
+
1. Fork it ( https://github.com/databyday/gtfs-meta/fork )
|
160
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
161
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
162
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
163
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
require "bundler/gem_tasks"
|
data/gtfs-meta.gemspec
ADDED
@@ -0,0 +1,25 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'gtfs/meta/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "gtfs-meta"
|
8
|
+
spec.version = GTFS::Meta::VERSION
|
9
|
+
spec.authors = ["MJ Rossetti"]
|
10
|
+
spec.email = ["s2t2mail@gmail.com"]
|
11
|
+
spec.summary = %q{A GTFS data and metadata manager for ActiveRecord.}
|
12
|
+
spec.description = %q{Extracts feed data, manages feed versions, and extends the feed specification to include feed metadata.}
|
13
|
+
spec.homepage = "https://github.com/databyday/gtfs-meta"
|
14
|
+
spec.license = "MIT"
|
15
|
+
|
16
|
+
spec.files = `git ls-files -z`.split("\x0")
|
17
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
+
spec.require_paths = ["lib"]
|
20
|
+
|
21
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
22
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
23
|
+
spec.add_dependency "activerecord", "~> 4.0.2"
|
24
|
+
spec.add_dependency "gtfs", "~> 0.2.4"
|
25
|
+
end
|
data/lib/gtfs.rb
ADDED
data/lib/gtfs/meta.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
require "gtfs"
|
2
|
+
|
3
|
+
module GTFS
|
4
|
+
module Meta
|
5
|
+
MESSAGE = "open transit data ftw"
|
6
|
+
GTFS_DATA_DIRECTORY = "db/gtfs" #todo: allow configuration
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
require "gtfs/meta/version"
|
11
|
+
|
12
|
+
require "gtfs/meta/migrations/create_publishers"
|
13
|
+
require "gtfs/meta/migrations/create_feeds"
|
14
|
+
require "gtfs/meta/models/publisher"
|
15
|
+
require "gtfs/meta/models/feed"
|
16
|
+
require "gtfs/meta/workers/feed_seeder"
|
17
|
+
|
18
|
+
require "gtfs/meta/migrations/create_feed_versions"
|
19
|
+
require "gtfs/meta/migrations/create_feed_version_checks"
|
20
|
+
require "gtfs/meta/models/feed_version"
|
21
|
+
require "gtfs/meta/models/feed_version_check"
|
22
|
+
require "gtfs/meta/workers/feed_manager"
|
File without changes
|
File without changes
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeedVersionChecks < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feed_version_checks do |t|
|
8
|
+
t.integer :feed_version_id, :null => false
|
9
|
+
t.string :status
|
10
|
+
t.timestamps
|
11
|
+
end
|
12
|
+
|
13
|
+
add_index :feed_version_checks, :feed_version_id
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeedVersions < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feed_versions do |t|
|
8
|
+
t.integer :feed_id, :null => false
|
9
|
+
t.string :etag, :null => false
|
10
|
+
t.datetime :last_modified_at, :null => false
|
11
|
+
t.text :file_names
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
|
15
|
+
add_index :feed_versions, :feed_id
|
16
|
+
add_index :feed_versions, [:feed_id, :etag, :last_modified_at], :unique => true, :name => "version_composite_key"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreateFeeds < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :feeds do |t|
|
8
|
+
t.integer :publisher_id, :null => false
|
9
|
+
t.string :source_url, :null => false
|
10
|
+
t.string :source_title, :null => false
|
11
|
+
t.timestamps
|
12
|
+
end
|
13
|
+
|
14
|
+
add_index :feeds, :publisher_id
|
15
|
+
add_index :feeds, :source_url, :unique => true
|
16
|
+
add_index :feeds, [:publisher_id, :source_title], :unique => true, :name => "publisher_unique_source_titles"
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class CreatePublishers < ActiveRecord::Migration
|
4
|
+
#TODO: re-construct this migration from a custom generator
|
5
|
+
|
6
|
+
def change
|
7
|
+
create_table :publishers do |t|
|
8
|
+
t.string :name
|
9
|
+
t.string :url, :null => false
|
10
|
+
t.string :feeds_url
|
11
|
+
t.string :email_address
|
12
|
+
t.timestamps
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
module GTFS
|
4
|
+
module Meta
|
5
|
+
class Feed < ActiveRecord::Base
|
6
|
+
#TODO: re-construct this model from a custom generator
|
7
|
+
|
8
|
+
belongs_to :publisher, :inverse_of => :feeds, :class_name => "GTFS::Meta::Publisher"
|
9
|
+
|
10
|
+
has_many :versions, :inverse_of => :feed, :class_name => "GTFS::Meta::FeedVersion"
|
11
|
+
|
12
|
+
validate :source_zipness_validator
|
13
|
+
|
14
|
+
def source_zipness_validator
|
15
|
+
errors.add(:source_url, "is not a zip") unless (source_url && source_file_name.ends_with?(".zip"))
|
16
|
+
end
|
17
|
+
|
18
|
+
def source_host_name
|
19
|
+
URI(source_url).hostname.split(".").reverse[1]
|
20
|
+
end
|
21
|
+
|
22
|
+
def source_file_name
|
23
|
+
source_url.split("/").last
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedVersion < ActiveRecord::Base
|
4
|
+
#TODO: re-construct this model from a custom generator
|
5
|
+
|
6
|
+
belongs_to :feed, :inverse_of => :versions, :class_name => "GTFS::Meta::Feed"
|
7
|
+
|
8
|
+
serialize :file_names, Array
|
9
|
+
|
10
|
+
def destination_path
|
11
|
+
"#{GTFS_DATA_DIRECTORY}/publishers/#{feed.source_host_name}/feeds/#{feed.source_title.downcase.gsub("-","").gsub(" ","")}/versions/#{etag}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedManager
|
4
|
+
FEED_FILE_NAMES = GTFS::Source::ENTITIES.map{|e| e.name == "Agency" ? e.singular_name.to_s.concat(".txt") : e.name.to_s.concat(".txt")}
|
5
|
+
|
6
|
+
def self.perform(options = {})
|
7
|
+
feeds = options[:feeds] || GTFS::Meta::Feed.all
|
8
|
+
|
9
|
+
feeds.find_each do |feed|
|
10
|
+
begin
|
11
|
+
# CHECK VERSION
|
12
|
+
|
13
|
+
uri = URI.parse(feed.source_url)
|
14
|
+
response = Net::HTTP.get_response(uri)
|
15
|
+
etag = response.to_hash["etag"].first.gsub!(/[^0-9A-Za-z]/, '')
|
16
|
+
last_modified_at = response.to_hash["last-modified"].first.try(:to_datetime)
|
17
|
+
|
18
|
+
raise EtagError.new(etag) unless etag.is_a?(String)
|
19
|
+
raise LastModifiedError.new(last_modified_at) unless last_modified_at.is_a?(DateTime)
|
20
|
+
feed_version = GTFS::Meta::FeedVersion.where({
|
21
|
+
:feed_id => feed.id,
|
22
|
+
:etag => etag,
|
23
|
+
:last_modified_at => last_modified_at
|
24
|
+
}).first_or_create
|
25
|
+
feed_version_check = GTFS::Meta::FeedVersionCheck.create({
|
26
|
+
:feed_version_id => feed_version.id,
|
27
|
+
:status => "IN PROGRESS"
|
28
|
+
})
|
29
|
+
|
30
|
+
# MANAGE FILES
|
31
|
+
|
32
|
+
FileUtils.mkdir_p(feed_version.destination_path)
|
33
|
+
|
34
|
+
zip_destination_path = "#{feed_version.destination_path}/#{feed.source_file_name}"
|
35
|
+
unless File.exist?(zip_destination_path)
|
36
|
+
File.open(zip_destination_path, "wb") do |zip_file|
|
37
|
+
zip_file.write response.body
|
38
|
+
end
|
39
|
+
raise SourceExtractionError.new(zip_destination_path) unless File.exist?(zip_destination_path)
|
40
|
+
end
|
41
|
+
|
42
|
+
Zip::File.open(zip_destination_path) do |zip_file|
|
43
|
+
zip_file.each do |entry|
|
44
|
+
begin
|
45
|
+
entry_name = entry.name
|
46
|
+
raise InvalidEntryName.new(entry_name) unless FEED_FILE_NAMES.include?(entry_name)
|
47
|
+
feed_version.file_names << entry_name unless feed_version.file_names.include?(entry_name)
|
48
|
+
|
49
|
+
feed_file_path = "#{feed_version.destination_path}/#{entry_name}"
|
50
|
+
raise FeedFileExists.new(feed_file_path) if File.exist?(feed_file_path)
|
51
|
+
|
52
|
+
entry.extract(feed_file_path)
|
53
|
+
raise FeedFileExtractionError.new(feed_file_path) unless File.exist?(feed_file_path)
|
54
|
+
rescue InvalidEntryName => e
|
55
|
+
next
|
56
|
+
rescue FeedFileExists => e
|
57
|
+
next
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
FileUtils.rm(zip_destination_path)
|
63
|
+
raise SourceRemovalError.new(zip_destination_path) if File.exist?(zip_destination_path)
|
64
|
+
|
65
|
+
feed_version.save!
|
66
|
+
feed_version_check.update_attributes(:status => "SUCCESS")
|
67
|
+
rescue => e
|
68
|
+
puts "#{e.class} -- #{e.message}"
|
69
|
+
feed_version_check.update_attributes(:status => "FAILURE") if feed_version_check
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class EtagError < StandardError
|
75
|
+
end
|
76
|
+
|
77
|
+
class LastModifiedError < StandardError
|
78
|
+
end
|
79
|
+
|
80
|
+
class SourceExtractionError < StandardError
|
81
|
+
end
|
82
|
+
|
83
|
+
class InvalidEntryName < StandardError
|
84
|
+
end
|
85
|
+
|
86
|
+
class FeedFileExists < StandardError
|
87
|
+
end
|
88
|
+
|
89
|
+
class FeedFileExtractionError < StandardError
|
90
|
+
end
|
91
|
+
|
92
|
+
class SourceRemovalError < StandardError
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module GTFS
|
2
|
+
module Meta
|
3
|
+
class FeedSeeder
|
4
|
+
def self.perform(options = {})
|
5
|
+
publishers = options[:publishers]
|
6
|
+
feeds = options[:feeds]
|
7
|
+
|
8
|
+
GTFS::Meta::Publisher.first_or_create(publishers)
|
9
|
+
GTFS::Meta::Feed.first_or_create(feeds)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gtfs-meta
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- MJ Rossetti
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-11-23 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '1.7'
|
22
|
+
type: :development
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.7'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '10.0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '10.0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: activerecord
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 4.0.2
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 4.0.2
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: gtfs
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ~>
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 0.2.4
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ~>
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.2.4
|
78
|
+
description: Extracts feed data, manages feed versions, and extends the feed specification
|
79
|
+
to include feed metadata.
|
80
|
+
email:
|
81
|
+
- s2t2mail@gmail.com
|
82
|
+
executables: []
|
83
|
+
extensions: []
|
84
|
+
extra_rdoc_files: []
|
85
|
+
files:
|
86
|
+
- .gitignore
|
87
|
+
- Gemfile
|
88
|
+
- LICENSE.txt
|
89
|
+
- README.md
|
90
|
+
- Rakefile
|
91
|
+
- gtfs-meta.gemspec
|
92
|
+
- lib/gtfs.rb
|
93
|
+
- lib/gtfs/meta.rb
|
94
|
+
- lib/gtfs/meta/generators/.gitkeep
|
95
|
+
- lib/gtfs/meta/migrations/.gitkeep
|
96
|
+
- lib/gtfs/meta/migrations/create_feed_version_checks.rb
|
97
|
+
- lib/gtfs/meta/migrations/create_feed_versions.rb
|
98
|
+
- lib/gtfs/meta/migrations/create_feeds.rb
|
99
|
+
- lib/gtfs/meta/migrations/create_publishers.rb
|
100
|
+
- lib/gtfs/meta/models/feed.rb
|
101
|
+
- lib/gtfs/meta/models/feed_version.rb
|
102
|
+
- lib/gtfs/meta/models/feed_version_check.rb
|
103
|
+
- lib/gtfs/meta/models/publisher.rb
|
104
|
+
- lib/gtfs/meta/version.rb
|
105
|
+
- lib/gtfs/meta/workers/feed_manager.rb
|
106
|
+
- lib/gtfs/meta/workers/feed_seeder.rb
|
107
|
+
homepage: https://github.com/databyday/gtfs-meta
|
108
|
+
licenses:
|
109
|
+
- MIT
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
none: false
|
116
|
+
requirements:
|
117
|
+
- - ! '>='
|
118
|
+
- !ruby/object:Gem::Version
|
119
|
+
version: '0'
|
120
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
requirements: []
|
127
|
+
rubyforge_project:
|
128
|
+
rubygems_version: 1.8.23
|
129
|
+
signing_key:
|
130
|
+
specification_version: 3
|
131
|
+
summary: A GTFS data and metadata manager for ActiveRecord.
|
132
|
+
test_files: []
|