cid 0.3.1 → 0.3.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NWM0ZjJlZDdjNWE5ZTAwOWRmYzg2MDYzODcyN2JlMWU3ODFiNGY3Ng==
+    MzM0MTcwZTBkMjhkYjBjMDQyZjliMzk0NjMxOWIxNGU0MmE0OGZhNg==
   data.tar.gz: !binary |-
-    MTFhMDE5N2RjMWI5YThkMjBiOGQ2NzM5OGNlMDU3N2M0ZWUwNTlkYQ==
+    ZTA4OTEwYWM3YmQ5ZDlhZjZlNDVhNWE5ZjU2ZDc2MWUwNjhjMjM1Mw==
 SHA512:
   metadata.gz: !binary |-
-    ZjZmZmVhMGY1ZDZhZDA5NDdhNWNiMjkxNDk1ZTM2YjBkYzc5MzJiMzBlNzQ0
-    OWMzYmVhODFjZDI2ZmU4MmUxZmYzYTRjMmFkZGU1MjIzYThiYjkxYTI1ODMy
-    YjQyNzhlNjFlMjM5ZGM4ODlmMjhjOGViM2E3NmFkODkyZTMyZDU=
+    YjdiNTA1NTNiNmUyODFiNWJlNTE3YWQ4OTUxOTkwYWQ1YWM4ZDA3NzA3Njcw
+    ODY1YWU2NWIxNDhlMDRiODk3N2MyYmE3MTM2Nzk5MDU0YmVjMDFhNDVlMWFh
+    MDdkOGIxYWE4ZmMxNmE5ZjZkNGY3NGU3NmJiMjA3NDg1YzRmNWM=
   data.tar.gz: !binary |-
-    ODk4NmVhYzMxZjI1NGVjNjhlODI3ODg4YzJkOTE4ZTEzMTYzOWUzMWUwNTdm
-    ZjRkMWMxMWFlNmMwMDRhZWMyOWY3NTZjMGY0OGIzZTE0N2U2ZThmODk3ZDEy
-    YmNkZWFiNTNiZjA2MmYwMGVjYjA5MmRkYTYyODkzMzVkMDE0MGE=
+    N2NmZmJhZDkxNmJiYzg1MTYyNjZlYjFmZGZhOWUwYmY1ZGI4YzcyYmU4MWY3
+    YmEwNWE5YWUwOWViZjIyYTIyYzM1MDcyMTYzNDJiZDIxOGIzOGQ4ZjBjOTUx
+    YTk1ZDA4MGQzZjE0MjU3MDcxZDUyZjQwZjc3ZDFlMTg4M2JhYjc=

data/README.md

@@ -101,7 +101,7 @@ If you just want to skip the GitHub push altogether, just run
 
 Obviously, Cid is at its most powerful when used in a CI build. To get Cid working in Travis, simply run:
 
-  cid bootstrap --github-token=YOUR_TOKEN_HERE
+  	cid bootstrap --github-token=YOUR_TOKEN_HERE
 
 This will create a `.travis.yml` to your repo and add your encrypted Github key.
 
@@ -115,7 +115,7 @@ If the branch is master, it will also generate a new `datapackage.json` and push
 If you would rather generate the `datapackage.json` on a different branch (for example, `gh-pages`),
 simply add the option `branch` like so:
 
-  cid bootstrap --github-token=YOUR_TOKEN_HERE --branch=gh-pages
+  	cid bootstrap --github-token=YOUR_TOKEN_HERE --branch=gh-pages
 
 You can also do this manually if you prefer - simply add a `.travis.yml` file to your repo
 that looks a bit like this:

data/lib/cid/datapackage.rb

@@ -17,25 +17,34 @@ module Cid
     end
 
     def create
-      # Clear out all the resources
-      @datapackage["resources"] = []
 
       paths = Dir.glob("#{@path}/*/")
 
       paths.each do |path|
-        schema = JSON.parse(File.new(Dir["#{path}/schema.json"][0]).read)
+
+        if File.file?("#{path}schema.json")
+          schema = JSON.parse(File.new(Dir["#{path}schema.json"][0]).read)
+        else
+          schema = nil
+        end
 
         Dir["#{path}*.csv"].each do |csv|
-          @datapackage["resources"] << {
+          ref = csv.split("/").last(2).join("/")
+
+          schema = schema_for_file(ref, true) rescue nil if schema.nil?
+
+          resource = {
             name: File.basename(csv, ".csv"),
-            path: csv.split("/").last(2).join("/"),
+            path: ref,
             format: "csv",
             mediatype: "text/csv",
-            bytes: File.size(csv),
-            schema: {
-                fields: schema["fields"]
-              }
+            bytes: File.size(csv)
           }
+
+          resource[:schema] = { fields: schema["fields"] } unless schema.nil?
+
+          @datapackage["resources"].reject! { |r| r["path"] == ref }
+          @datapackage["resources"] << resource
         end
       end
 
@@ -52,13 +61,14 @@ module Cid
       git.publish
     end
 
-    def schema_for_file(path)
+    def schema_for_file(path, json = false)
       begin
         schema = @datapackage["resources"].select { |r| r["path"] == path }.first["schema"]
-        Csvlint::Schema.from_json_table(nil, schema)
+        return Csvlint::Schema.from_json_table(nil, schema) if json === false
       rescue NoMethodError
         nil
       end
+      return schema
     end
 
   end

data/lib/cid/version.rb

@@ -1,3 +1,3 @@
 module Cid
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/spec/datapackage_spec.rb

@@ -2,22 +2,50 @@ require 'spec_helper'
 
 describe Cid::Datapackage do
 
-  it "recrates the original datapackage.rb" do
-    datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'valid'))
-    datapackage.create
+  context "when a schema is contained within a folder" do
+
+    it "recrates the original datapackage.rb" do
+      datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'valid'))
+      datapackage.create
+
+      JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
+    end
+
+    it "creates a datapackage.rb with a new file" do
+      datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'multiple_files'))
+      datapackage.create
+
+      package = JSON.parse(datapackage.json)
+
+      package["resources"].count.should == 2
+      package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
+      package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
+    end
 
-    JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
   end
 
-  it "creates a datapackage.rb with a new file" do
-    datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'multiple_files'))
-    datapackage.create
+  context "when a schema is contained in the datapackage" do
+
+    it "recrates the original datapackage.rb" do
+      datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'datapackage_valid'))
+      datapackage.create
+
+      JSON.parse(datapackage.json).should == JSON.parse(File.read(File.join(File.dirname(__FILE__), 'fixtures', 'valid', 'datapackage.json')))
+    end
+
+    it "creates a datapackage.rb with a new file" do
+      datapackage = Cid::Datapackage.new(File.join(File.dirname(__FILE__), 'fixtures', 'datapackage_multiple_files'))
+      datapackage.create
+
+      package = JSON.parse(datapackage.json)
+
+      package["resources"].count.should == 2
+      package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
+      package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
 
-    package = JSON.parse(datapackage.json)
+      package["resources"].select { |r| r["path"] == "votes/votes-2.csv" }[0]["schema"].should_not be_nil
+    end
 
-    package["resources"].count.should == 2
-    package["resources"].flat_map(&:to_a).should include(["name", "votes-2"])
-    package["resources"].flat_map(&:to_a).should include(["path", "votes/votes-2.csv"])
   end
 
 end

data/spec/fixtures/datapackage_multiple_files/datapackage.json

@@ -0,0 +1,71 @@
+{
+    "name": "euro-elections",
+    "title": "",
+    "homepage": "",
+    "version": "0.1.0",
+    "license": "PDDL-1.0",
+    "resources": [
+        {
+            "name": "votes-1",
+            "path": "votes/votes-1.csv",
+            "format": "csv",
+            "mediatype": "text/csv",
+            "bytes": 1752,
+            "schema": {
+                "fields": [
+                    {
+                        "name": "Poll Date",
+                        "type": "date",
+                        "description": "Date the poll took place",
+                        "format": "yyyy-mm-dd"
+                    },
+                    {
+                        "name": "Area",
+                        "type": "string",
+                        "description": "The name of the area which the result is for",
+                        "required": true
+                    },
+                    {
+                        "name": "Area Code",
+                        "type": "number",
+                        "description": "The Office of National Statistics code for the area",
+                        "pattern": "E1500000(0-9)",
+                        "required": true
+                    },
+                    {
+                        "name": "Area URL",
+                        "type": "string",
+                        "description": "",
+                        "required": true,
+                        "pattern": "http://statistics.data.gov.uk/doc/statistical-geography/E1500000(0-9)"
+                    },
+                    {
+                        "name": "Party",
+                        "type": "string",
+                        "description": "",
+                        "required": true
+                    },
+                    {
+                        "name": "Party ID",
+                        "type": "string",
+                        "description": "",
+                        "required": true,
+                        "pattern": "PPm? [0-9]+"
+                    },
+                    {
+                        "name": "Votes",
+                        "type": "integer",
+                        "description": ""
+                    },
+                    {
+                        "name": "Ballots Rejected",
+                        "type": "integer",
+                        "description": ""
+                    }
+                ]
+            }
+        }
+    ],
+    "description": "",
+    "repository": "git://github.com/theodi/euro-elections.git"
+}

data/spec/fixtures/datapackage_multiple_files/votes/votes-1.csv

@@ -0,0 +1,13 @@
+Poll Date,Area,Area Code,Area URL,Party,Party ID,Votes,Ballots Rejected
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,British National Party ,PP 106,121967,9216
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Christian Party,PP 1992,18784,9217
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Conservative Party ,PP 52,396847,9218
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,English Democrats Party,PP 17,32455,9219
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Jury Team,PP 999,8721,9220
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Liberal Democrats,PP 90,170246,9221
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,No2EU: Yes to Democracy,PP 2164,13415,9222
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Pro Democracy: Libertas.eu ,PP 999,6961,9223
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Socialist Labour Party,PP 73,14724,9224
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Green Party,PP 63,88244,9225
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Labour Party,PP 53,240201,9226
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,United Kingdom Independence Party,PP 85,300471,9227

data/spec/fixtures/datapackage_multiple_files/votes/votes-2.csv

@@ -0,0 +1,13 @@
+Poll Date,Area,Area Code,Area URL,Party,Party ID,Votes,Ballots Rejected
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,British National Party ,PP 106,121967,9216
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Christian Party,PP 1992,18784,9217
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Conservative Party ,PP 52,396847,9218
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,English Democrats Party,PP 17,32455,9219
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Jury Team,PP 999,8721,9220
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Liberal Democrats,PP 90,170246,9221
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,No2EU: Yes to Democracy,PP 2164,13415,9222
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Pro Democracy: Libertas.eu ,PP 999,6961,9223
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,Socialist Labour Party,PP 73,14724,9224
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Green Party,PP 63,88244,9225
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,The Labour Party,PP 53,240201,9226
+2009-06-04,West Midlands,E15000005,http://statistics.data.gov.uk/doc/statistical-geography/E15000005,United Kingdom Independence Party,PP 85,300471,9227

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cid
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - pezholio
@@ -340,6 +340,9 @@ files:
 - spec/cassettes/Cid_Publish/should_be_able_to_get_latest_commit_SHA_on_a_branch.yml
 - spec/cassettes/Cid_Publish/should_push_to_the_default_branch.yml
 - spec/datapackage_spec.rb
+- spec/fixtures/datapackage_multiple_files/datapackage.json
+- spec/fixtures/datapackage_multiple_files/votes/votes-1.csv
+- spec/fixtures/datapackage_multiple_files/votes/votes-2.csv
 - spec/fixtures/datapackage_valid/datapackage.json
 - spec/fixtures/datapackage_valid/votes/votes-1.csv
 - spec/fixtures/invalid/votes/schema.json
@@ -399,6 +402,9 @@ test_files:
 - spec/cassettes/Cid_Publish/should_be_able_to_get_latest_commit_SHA_on_a_branch.yml
 - spec/cassettes/Cid_Publish/should_push_to_the_default_branch.yml
 - spec/datapackage_spec.rb
+- spec/fixtures/datapackage_multiple_files/datapackage.json
+- spec/fixtures/datapackage_multiple_files/votes/votes-1.csv
+- spec/fixtures/datapackage_multiple_files/votes/votes-2.csv
 - spec/fixtures/datapackage_valid/datapackage.json
 - spec/fixtures/datapackage_valid/votes/votes-1.csv
 - spec/fixtures/invalid/votes/schema.json