nistbib 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +4 -4
- data/lib/nistbib/hit_collection.rb +4 -5
- data/lib/nistbib/nist_bibliography.rb +1 -1
- data/lib/nistbib/scrapper.rb +3 -1
- data/lib/nistbib/version.rb +1 -1
- data/nistbib.gemspec +1 -1
- metadata +3 -4
- data/lib/nistbib/workers_pool.rb +0 -41
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3fd84906d23f136d265529518acb196d88634951
|
4
|
+
data.tar.gz: 3f061f6e0fc98fe427e21191ec04625753ebf6e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c350a8519d63c55d12abd1d368d805dbfce664385056e3c6562e0f2659c0cfb56aec36eed20d5588ea5ced1479521821a69aa6fdbd1e3cc27a413c3b6c554c5f
|
7
|
+
data.tar.gz: 608506b77c1051759e51bfd65c45e9c85723b6baf98eeb04916623023194c55e098e150121baa68ca1ca98dae0980de20f80e9ba4034f6929dc6bc23cdf49f6d
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
nistbib (0.1.
|
5
|
-
relaton-bib (~> 0.1.
|
4
|
+
nistbib (0.1.9)
|
5
|
+
relaton-bib (~> 0.1.6)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
@@ -32,9 +32,9 @@ GEM
|
|
32
32
|
pry-byebug (3.7.0)
|
33
33
|
byebug (~> 11.0)
|
34
34
|
pry (~> 0.10)
|
35
|
-
public_suffix (3.0
|
35
|
+
public_suffix (3.1.0)
|
36
36
|
rake (10.5.0)
|
37
|
-
relaton-bib (0.1.
|
37
|
+
relaton-bib (0.1.6)
|
38
38
|
addressable
|
39
39
|
nokogiri (~> 1.8.4)
|
40
40
|
rspec (3.8.0)
|
@@ -1,15 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require 'nistbib/workers_pool'
|
3
|
+
require "nistbib/hit"
|
5
4
|
require "addressable/uri"
|
6
|
-
require
|
5
|
+
require "open-uri"
|
7
6
|
|
8
7
|
module NistBib
|
9
8
|
# Page of hit collection.
|
10
9
|
class HitCollection < Array
|
11
10
|
|
12
|
-
DOMAIN =
|
11
|
+
DOMAIN = "https://csrc.nist.gov"
|
13
12
|
|
14
13
|
# @return [TrueClass, FalseClass]
|
15
14
|
attr_reader :fetched
|
@@ -61,7 +60,7 @@ module NistBib
|
|
61
60
|
|
62
61
|
# @return [Iecbib::HitCollection]
|
63
62
|
def fetch
|
64
|
-
workers = WorkersPool.new 4
|
63
|
+
workers = RelatonBib::WorkersPool.new 4
|
65
64
|
workers.worker(&:fetch)
|
66
65
|
each do |hit|
|
67
66
|
workers << hit
|
@@ -116,7 +116,7 @@ module NistBib
|
|
116
116
|
end
|
117
117
|
|
118
118
|
def fetch_pages(s, n)
|
119
|
-
workers = WorkersPool.new n
|
119
|
+
workers = RelatonBib::WorkersPool.new n
|
120
120
|
workers.worker { |w| { i: w[:i], hit: w[:hit].fetch } }
|
121
121
|
s.each_with_index { |hit, i| workers << { i: i, hit: hit } }
|
122
122
|
workers.end
|
data/lib/nistbib/scrapper.rb
CHANGED
@@ -5,10 +5,11 @@ module NistBib
|
|
5
5
|
class << self
|
6
6
|
DOMAIN = "https://csrc.nist.gov".freeze
|
7
7
|
|
8
|
+
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
9
|
+
|
8
10
|
# Parse page.
|
9
11
|
# @param hit_data [Hash]
|
10
12
|
# @return [Hash]
|
11
|
-
# rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
12
13
|
def parse_page(hit_data)
|
13
14
|
doc = get_page hit_data[:url]
|
14
15
|
|
@@ -33,6 +34,7 @@ module NistBib
|
|
33
34
|
commentperiod: fetch_commentperiod(doc),
|
34
35
|
)
|
35
36
|
end
|
37
|
+
# rubocop:enable Metrics/AbcSize, Metrics/MethodLength
|
36
38
|
|
37
39
|
private
|
38
40
|
|
data/lib/nistbib/version.rb
CHANGED
data/nistbib.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nistbib
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ribose Inc.
|
@@ -170,14 +170,14 @@ dependencies:
|
|
170
170
|
requirements:
|
171
171
|
- - "~>"
|
172
172
|
- !ruby/object:Gem::Version
|
173
|
-
version: 0.1.
|
173
|
+
version: 0.1.6
|
174
174
|
type: :runtime
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
|
-
version: 0.1.
|
180
|
+
version: 0.1.6
|
181
181
|
description: 'NistBib: retrive NIST standards.'
|
182
182
|
email:
|
183
183
|
- open.source@ribose.com
|
@@ -207,7 +207,6 @@ files:
|
|
207
207
|
- lib/nistbib/nist_bibliography.rb
|
208
208
|
- lib/nistbib/scrapper.rb
|
209
209
|
- lib/nistbib/version.rb
|
210
|
-
- lib/nistbib/workers_pool.rb
|
211
210
|
- lib/nistbib/xml_parser.rb
|
212
211
|
- lib/relaton/processor.rb
|
213
212
|
- nistbib.gemspec
|
data/lib/nistbib/workers_pool.rb
DELETED
@@ -1,41 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Workers poll.
|
4
|
-
class WorkersPool
|
5
|
-
attr_accessor :nb_hits
|
6
|
-
|
7
|
-
def initialize(num_workers = 2)
|
8
|
-
@num_workers = num_workers < 2 ? 2 : num_workers
|
9
|
-
@queue = SizedQueue.new(num_workers * 2)
|
10
|
-
@result = []
|
11
|
-
@nb_hits = 0
|
12
|
-
end
|
13
|
-
|
14
|
-
def worker(&block)
|
15
|
-
@threads = Array.new @num_workers do
|
16
|
-
Thread.new do
|
17
|
-
until (item = @queue.pop) == :END
|
18
|
-
@result << yield(item) if block
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def result
|
25
|
-
@threads.each(&:join)
|
26
|
-
@result
|
27
|
-
end
|
28
|
-
|
29
|
-
def <<(item)
|
30
|
-
@queue << item
|
31
|
-
self
|
32
|
-
end
|
33
|
-
|
34
|
-
def end
|
35
|
-
@num_workers.times { @queue << :END }
|
36
|
-
end
|
37
|
-
|
38
|
-
# def size
|
39
|
-
# @result.size
|
40
|
-
# end
|
41
|
-
end
|