ecfs 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -0
- data/lib/ecfs/solr_scrape_query.rb +8 -3
- data/lib/ecfs/version.rb +1 -1
- data/test/test_solr_scrape.rb +34 -0
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa5da38c0c5348fa0c845d28c0f9c2516cf33bb3
|
4
|
+
data.tar.gz: 30aed783337f289d994d490d26240bb43b6266e4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e91e0878401399171a6f346d72049a3d3c71975d2a6ca1671e7775a8f49e5a7bfac62e84765a7ec27b500e95599c2a8df77e684fe3e4885ee2e0a1dc9446432e
|
7
|
+
data.tar.gz: 0fd4b2971174f2c3317016d85282a056b046ea3f06702349b92de6b5f50804ddf35591aa0e41f10f70e260fbfe28d009e2e2147cf3e8fd1eaa73b8ab960d4d41
|
data/README.md
CHANGED
@@ -256,6 +256,23 @@ p filings.first
|
|
256
256
|
}
|
257
257
|
```
|
258
258
|
|
259
|
+
Options:
|
260
|
+
|
261
|
+
```ruby
|
262
|
+
filings = ECFS::SolrScrapeQuery.new.tap do |q|
|
263
|
+
q.docket_number = '12-83'
|
264
|
+
|
265
|
+
# a minimum date, inclusive. mm/dd/yyyy
|
266
|
+
q.received_min_date = '03/30/2012'
|
267
|
+
|
268
|
+
# an after_scrape block
|
269
|
+
q.after_scrape = Proc.new do |filings|
|
270
|
+
p "Fetched asynchronyously: #{filings.length}"
|
271
|
+
end
|
272
|
+
# This is handy for large scrapes.
|
273
|
+
end.get
|
274
|
+
```
|
275
|
+
|
259
276
|
### Daily Releases
|
260
277
|
|
261
278
|
This feature parses these types of pages: http://transition.fcc.gov/Daily_Releases/Daily_Business/2014/db0917/.
|
@@ -7,8 +7,9 @@ module ECFS
|
|
7
7
|
class SolrScrapeQuery
|
8
8
|
attr_accessor :docket_number
|
9
9
|
attr_accessor :received_min_date
|
10
|
+
attr_accessor :after_scrape
|
10
11
|
|
11
|
-
def filings_from_docket_number(docket_number, start=0, received_min_date=nil)
|
12
|
+
def filings_from_docket_number(docket_number, start=0, received_min_date=nil, after_scrape=nil)
|
12
13
|
url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{docket_number}&dir=asc&start=#{start}"
|
13
14
|
|
14
15
|
if received_min_date
|
@@ -46,6 +47,10 @@ module ECFS
|
|
46
47
|
}
|
47
48
|
end
|
48
49
|
|
50
|
+
if after_scrape
|
51
|
+
after_scrape.call(filings)
|
52
|
+
end
|
53
|
+
|
49
54
|
return filings, total
|
50
55
|
end
|
51
56
|
|
@@ -72,7 +77,7 @@ module ECFS
|
|
72
77
|
url = "http://apps.fcc.gov/ecfs/solr/search?sort=dateRcpt&proceeding=#{@docket_number}&dir=asc&start=0"
|
73
78
|
filings = []
|
74
79
|
|
75
|
-
first_page_of_filings, total = filings_from_docket_number(@docket_number, 0, @received_min_date)
|
80
|
+
first_page_of_filings, total = filings_from_docket_number(@docket_number, 0, @received_min_date, @after_scrape)
|
76
81
|
|
77
82
|
pages = (total.to_f/20.0).ceil.to_i.times.map {|n| n*20} # divide, round up, then map *20
|
78
83
|
pages.shift
|
@@ -80,7 +85,7 @@ module ECFS
|
|
80
85
|
filings.concat first_page_of_filings
|
81
86
|
|
82
87
|
pages.each do |page|
|
83
|
-
filings.concat filings_from_docket_number(@docket_number, page, @received_min_date)[0]
|
88
|
+
filings.concat filings_from_docket_number(@docket_number, page, @received_min_date, @after_scrape)[0]
|
84
89
|
end
|
85
90
|
|
86
91
|
filings.each do |filing|
|
data/lib/ecfs/version.rb
CHANGED
data/test/test_solr_scrape.rb
CHANGED
@@ -33,5 +33,39 @@ class TestSolrScrape < MiniTest::Unit::TestCase
|
|
33
33
|
assert filing_date > min_date
|
34
34
|
end
|
35
35
|
end
|
36
|
+
|
37
|
+
class FakeArrayThing
|
38
|
+
def initialize
|
39
|
+
@filings = []
|
40
|
+
end
|
41
|
+
|
42
|
+
def concat(filings)
|
43
|
+
@filings.concat(filings)
|
44
|
+
end
|
45
|
+
|
46
|
+
def filings
|
47
|
+
@filings
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_after_scrape
|
52
|
+
VCR.use_cassette('solr_cassette') do
|
53
|
+
|
54
|
+
@fake_array_thing = FakeArrayThing.new
|
55
|
+
|
56
|
+
filings = ECFS::SolrScrapeQuery.new.tap do |q|
|
57
|
+
q.docket_number = '12-83'
|
58
|
+
q.after_scrape = Proc.new do |filings|
|
59
|
+
@fake_array_thing.concat(filings)
|
60
|
+
end
|
61
|
+
end.get
|
62
|
+
|
63
|
+
assert filings.first.is_a?(Hash)
|
64
|
+
assert filings.first.has_key?('docket_number')
|
65
|
+
assert filings.first.has_key?('citation')
|
66
|
+
|
67
|
+
assert_equal filings.length, @fake_array_thing.filings.length
|
68
|
+
end
|
69
|
+
end
|
36
70
|
|
37
71
|
end
|