cid 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +2 -2
- data/lib/cid/datapackage.rb +21 -11
- data/lib/cid/version.rb +1 -1
- data/spec/datapackage_spec.rb +39 -11
- data/spec/fixtures/datapackage_multiple_files/datapackage.json +71 -0
- data/spec/fixtures/datapackage_multiple_files/votes/votes-1.csv +13 -0
- data/spec/fixtures/datapackage_multiple_files/votes/votes-2.csv +13 -0
- metadata +7 -1
checksums.yaml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
!binary "U0hBMQ==":
|
|
3
3
|
metadata.gz: !binary |-
|
|
4
|
-
|
|
4
|
+
MzM0MTcwZTBkMjhkYjBjMDQyZjliMzk0NjMxOWIxNGU0MmE0OGZhNg==
|
|
5
5
|
data.tar.gz: !binary |-
|
|
6
|
-
|
|
6
|
+
ZTA4OTEwYWM3YmQ5ZDlhZjZlNDVhNWE5ZjU2ZDc2MWUwNjhjMjM1Mw==
|
|
7
7
|
SHA512:
|
|
8
8
|
metadata.gz: !binary |-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
YjdiNTA1NTNiNmUyODFiNWJlNTE3YWQ4OTUxOTkwYWQ1YWM4ZDA3NzA3Njcw
|
|
10
|
+
ODY1YWU2NWIxNDhlMDRiODk3N2MyYmE3MTM2Nzk5MDU0YmVjMDFhNDVlMWFh
|
|
11
|
+
MDdkOGIxYWE4ZmMxNmE5ZjZkNGY3NGU3NmJiMjA3NDg1YzRmNWM=
|
|
12
12
|
data.tar.gz: !binary |-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
N2NmZmJhZDkxNmJiYzg1MTYyNjZlYjFmZGZhOWUwYmY1ZGI4YzcyYmU4MWY3
|
|
14
|
+
YmEwNWE5YWUwOWViZjIyYTIyYzM1MDcyMTYzNDJiZDIxOGIzOGQ4ZjBjOTUx
|
|
15
|
+
YTk1ZDA4MGQzZjE0MjU3MDcxZDUyZjQwZjc3ZDFlMTg4M2JhYjc=
|
data/README.md
CHANGED
|
@@ -101,7 +101,7 @@ If you just want to skip the GitHub push altogether, just run
|
|
|
101
101
|
|
|
102
102
|
Obviously, Cid is at its most powerful when used in a CI build. To get Cid working in Travis, simply run:
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
cid bootstrap --github-token=YOUR_TOKEN_HERE
|
|
105
105
|
|
|
106
106
|
This will create a `.travis.yml` to your repo and add your encrypted Github key.
|
|
107
107
|
|
|
@@ -115,7 +115,7 @@ If the branch is master, it will also generate a new `datapackage.json` and push
|
|
|
115
115
|
If you would rather generate the `datapackage.json` on a different branch (for example, `gh-pages`),
|
|
116
116
|
simply add the option `branch` like so:
|
|
117
117
|
|
|
118
|
-
|
|
118
|
+
cid bootstrap --github-token=YOUR_TOKEN_HERE --branch=gh-pages
|
|
119
119
|
|
|
120
120
|
You can also do this manually if you prefer - simply add a `.travis.yml` file to your repo
|
|
121
121
|
that looks a bit like this:
|
data/lib/cid/datapackage.rb
CHANGED
|
@@ -17,25 +17,34 @@ module Cid
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def create
|
|
20
|
-
# Clear out all the resources
|
|
21
|
-
@datapackage["resources"] = []
|
|
22
20
|
|
|
23
21
|
paths = Dir.glob("#{@path}/*/")
|
|
24
22
|
|
|
25
23
|
paths.each do |path|
|
|
26
|
-
|
|
24
|
+
|
|
25
|
+
if File.file?("#{path}schema.json")
|
|
26
|
+
schema = JSON.parse(File.new(Dir["#{path}schema.json"][0]).read)
|
|
27
|
+
else
|
|
28
|
+
schema = nil
|
|
29
|
+
end
|
|
27
30
|
|
|
28
31
|
Dir["#{path}*.csv"].each do |csv|
|
|
29
|
-
|
|
32
|
+
ref = csv.split("/").last(2).join("/")
|
|
33
|
+
|
|
34
|
+
schema = schema_for_file(ref, true) rescue nil if schema.nil?
|
|
35
|
+
|
|
36
|
+
resource = {
|
|
30
37
|
name: File.basename(csv, ".csv"),
|
|
31
|
-
path:
|
|
38
|
+
path: ref,
|
|
32
39
|
format: "csv",
|
|
33
40
|
mediatype: "text/csv",
|
|
34
|
-
bytes: File.size(csv)
|
|
35
|
-
schema: {
|
|
36
|
-
fields: schema["fields"]
|
|
37
|
-
}
|
|
41
|
+
bytes: File.size(csv)
|
|
38
42
|
}
|
|
43
|
+
|
|
44
|
+
resource[:schema] = { fields: schema["fields"] } unless schema.nil?
|
|
45
|
+
|
|
46
|
+
@datapackage["resources"].reject! { |r| r["path"] == ref }
|
|
47
|
+
@datapackage["resources"] << resource
|
|
39
48
|
end
|
|
40
49
|
end
|
|
41
50
|
|
|
@@ -52,13 +61,14 @@ module Cid
|
|
|
52
61
|
git.publish
|
|
53
62
|
end
|
|
54
63
|
|
|
55
|
-
def schema_for_file(path)
|
|
64
|
+
def schema_for_file(path, json = false)
|
|
56
65
|
begin
|
|
57
66
|
schema = @datapackage["resources"].select { |r| r["path"] == path }.first["schema"]
|
|
58
|
-
Csvlint::Schema.from_json_table(nil, schema)
|
|
67
|
+
return Csvlint::Schema.from_json_table(nil, schema) if json === false
|
|
59
68
|
rescue NoMethodError
|
|
60
69
|
nil
|
|
61
70
|
end
|
|
71
|
+
return schema
|
|
62
72
|
end
|
|
63
73
|
|
|
64
74
|
end
|
data/lib/cid/version.rb
CHANGED
data/spec/datapackage_spec.rb
CHANGED
|
@@ -2,22 +2,50 @@ require 'spec_helper'
|
|
|
2
2
|
|
|
3
3
|
describe Cid::Datapackage do
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
datapackage.
|
|
5
|
+
context "when a schema is contained within a folder" do
|
|
6
|
+
|
|
7
|
+
it "recrates the original datapackage.rb" do
|
|
8
|
+
datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'valid'))
|
|
9
|
+
datapackage.create
|
|
10
|
+
|
|
11
|
+
JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
it "creates a datapackage.rb with a new file" do
|
|
15
|
+
datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'multiple_files'))
|
|
16
|
+
datapackage.create
|
|
17
|
+
|
|
18
|
+
package = JSON.parse(datapackage.json)
|
|
19
|
+
|
|
20
|
+
package["resources"].count.should == 2
|
|
21
|
+
package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
|
|
22
|
+
package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
|
|
23
|
+
end
|
|
8
24
|
|
|
9
|
-
JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
|
|
10
25
|
end
|
|
11
26
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
datapackage.
|
|
27
|
+
context "when a schema is contained in the datapackage" do
|
|
28
|
+
|
|
29
|
+
it "recrates the original datapackage.rb" do
|
|
30
|
+
datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'datapackage_valid'))
|
|
31
|
+
datapackage.create
|
|
32
|
+
|
|
33
|
+
JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
it "creates a datapackage.rb with a new file" do
|
|
37
|
+
datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'datapackage_multiple_files'))
|
|
38
|
+
datapackage.create
|
|
39
|
+
|
|
40
|
+
package = JSON.parse(datapackage.json)
|
|
41
|
+
|
|
42
|
+
package["resources"].count.should == 2
|
|
43
|
+
package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
|
|
44
|
+
package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
|
|
15
45
|
|
|
16
|
-
|
|
46
|
+
package["resources"].select { |r| r["path"] == "votes/votes-2.csv" }[0]["schema"].should_not be_nil
|
|
47
|
+
end
|
|
17
48
|
|
|
18
|
-
package["resources"].count.should == 2
|
|
19
|
-
package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
|
|
20
|
-
package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
|
|
21
49
|
end
|
|
22
50
|
|
|
23
51
|
end
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "euro-elections",
|
|
3
|
+
"title": "",
|
|
4
|
+
"homepage": "",
|
|
5
|
+
"version": "0.1.0",
|
|
6
|
+
"license": "PDDL-1.0",
|
|
7
|
+
"resources": [
|
|
8
|
+
{
|
|
9
|
+
"name": "votes-1",
|
|
10
|
+
"path": "votes/votes-1.csv",
|
|
11
|
+
"format": "csv",
|
|
12
|
+
"mediatype": "text/csv",
|
|
13
|
+
"bytes": 1752,
|
|
14
|
+
"schema": {
|
|
15
|
+
"fields": [
|
|
16
|
+
{
|
|
17
|
+
"name": "Poll Date",
|
|
18
|
+
"type": "date",
|
|
19
|
+
"description": "Date the poll took place",
|
|
20
|
+
"format": "yyyy-mm-dd"
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"name": "Area",
|
|
24
|
+
"type": "string",
|
|
25
|
+
"description": "The name of the area which the result is for",
|
|
26
|
+
"required": true
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"name": "Area Code",
|
|
30
|
+
"type": "number",
|
|
31
|
+
"description": "The Office of National Statistics code for the area",
|
|
32
|
+
"pattern": "E1500000(0-9)",
|
|
33
|
+
"required": true
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"name": "Area URL",
|
|
37
|
+
"type": "string",
|
|
38
|
+
"description": "",
|
|
39
|
+
"required": true,
|
|
40
|
+
"pattern": "http://statistics.data.gov.uk/doc/statistical-geography/E1500000(0-9)"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"name": "Party",
|
|
44
|
+
"type": "string",
|
|
45
|
+
"description": "",
|
|
46
|
+
"required": true
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
"name": "Party ID",
|
|
50
|
+
"type": "string",
|
|
51
|
+
"description": "",
|
|
52
|
+
"required": true,
|
|
53
|
+
"pattern": "PPm? [0-9]+"
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"name": "Votes",
|
|
57
|
+
"type": "integer",
|
|
58
|
+
"description": ""
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"name": "Ballots Rejected",
|
|
62
|
+
"type": "integer",
|
|
63
|
+
"description": ""
|
|
64
|
+
}
|
|
65
|
+
]
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
],
|
|
69
|
+
"description": "",
|
|
70
|
+
"repository": "git://github.com/theodi/euro-elections.git"
|
|
71
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Poll Date,Area,Area Code,Area URL,Party,Party ID,Votes,Ballots Rejected
|
|
2
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,British National Party ,PP 106,121967,9216
|
|
3
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Christian Party,PP 1992,18784,9217
|
|
4
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Conservative Party ,PP 52,396847,9218
|
|
5
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,English Democrats Party,PP 17,32455,9219
|
|
6
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Jury Team,PP 999,8721,9220
|
|
7
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Liberal Democrats,PP 90,170246,9221
|
|
8
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,No2EU: Yes to Democracy,PP 2164,13415,9222
|
|
9
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Pro Democracy: Libertas.eu ,PP 999,6961,9223
|
|
10
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Socialist Labour Party,PP 73,14724,9224
|
|
11
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Green Party,PP 63,88244,9225
|
|
12
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Labour Party,PP 53,240201,9226
|
|
13
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,United Kingdom Independence Party,PP 85,300471,9227
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Poll Date,Area,Area Code,Area URL,Party,Party ID,Votes,Ballots Rejected
|
|
2
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,British National Party ,PP 106,121967,9216
|
|
3
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Christian Party,PP 1992,18784,9217
|
|
4
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Conservative Party ,PP 52,396847,9218
|
|
5
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,English Democrats Party,PP 17,32455,9219
|
|
6
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Jury Team,PP 999,8721,9220
|
|
7
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Liberal Democrats,PP 90,170246,9221
|
|
8
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,No2EU: Yes to Democracy,PP 2164,13415,9222
|
|
9
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Pro Democracy: Libertas.eu ,PP 999,6961,9223
|
|
10
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Socialist Labour Party,PP 73,14724,9224
|
|
11
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Green Party,PP 63,88244,9225
|
|
12
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Labour Party,PP 53,240201,9226
|
|
13
|
+
2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,United Kingdom Independence Party,PP 85,300471,9227
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cid
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- pezholio
|
|
@@ -340,6 +340,9 @@ files:
|
|
|
340
340
|
- spec/cassettes/Cid_Publish/should_be_able_to_get_latest_commit_SHA_on_a_branch.yml
|
|
341
341
|
- spec/cassettes/Cid_Publish/should_push_to_the_default_branch.yml
|
|
342
342
|
- spec/datapackage_spec.rb
|
|
343
|
+
- spec/fixtures/datapackage_multiple_files/datapackage.json
|
|
344
|
+
- spec/fixtures/datapackage_multiple_files/votes/votes-1.csv
|
|
345
|
+
- spec/fixtures/datapackage_multiple_files/votes/votes-2.csv
|
|
343
346
|
- spec/fixtures/datapackage_valid/datapackage.json
|
|
344
347
|
- spec/fixtures/datapackage_valid/votes/votes-1.csv
|
|
345
348
|
- spec/fixtures/invalid/votes/schema.json
|
|
@@ -399,6 +402,9 @@ test_files:
|
|
|
399
402
|
- spec/cassettes/Cid_Publish/should_be_able_to_get_latest_commit_SHA_on_a_branch.yml
|
|
400
403
|
- spec/cassettes/Cid_Publish/should_push_to_the_default_branch.yml
|
|
401
404
|
- spec/datapackage_spec.rb
|
|
405
|
+
- spec/fixtures/datapackage_multiple_files/datapackage.json
|
|
406
|
+
- spec/fixtures/datapackage_multiple_files/votes/votes-1.csv
|
|
407
|
+
- spec/fixtures/datapackage_multiple_files/votes/votes-2.csv
|
|
402
408
|
- spec/fixtures/datapackage_valid/datapackage.json
|
|
403
409
|
- spec/fixtures/datapackage_valid/votes/votes-1.csv
|
|
404
410
|
- spec/fixtures/invalid/votes/schema.json
|