cid 0.0.2 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 08cc9c8917f69e177896bea64e370d42af222bf7
+  data.tar.gz: 2a54b94ce968d84233d8f8de41883b214ca797a8
+SHA512:
+  metadata.gz: a6ad26c203b9c356578ed153e8cf04182e4d230182e2d07158e8ab41d24a820e4da6409bdac389935a32ace32b31ff28bd6d29a85a0fe0a06c372a6bea0355ab
+  data.tar.gz: 3da71f338651e9086f3c7d6a2f2f9219b6f7a867a5c3859a4f2902072175355b479e21380e830a2a224dff9981099e27c808ac83b447cb54ad6ce35ab74d7a0d

data/.gitignore

@@ -15,3 +15,5 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+
+/.env

data/.travis.yml

@@ -0,0 +1,13 @@
+rvm:
+- 2.0.0
+notifications:
+  irc:
+    channels:
+    - irc.freenode.net#theodi
+    template:
+    - '%{repository} %{branch} - %{message} %{build_url}'
+    on_success: change
+    on_failure: always
+env:
+  global:
+    secure: HZhBJjfU3C6ub01Ioong30m3NMAoT5re8XwDg3VQ86L2Z/lz/Zz7FU+cJvLcJJaO7Jj7UmZK0ov0kFxIesIJMV/KsMfmMOlJjTMz0K5g/C/NaTIKlZTY4Mg53oS4fh0ZNogtfkttOfL/JLyNS2buuSEuMtFtC0+DXlqklQ/JT3w=

data/Gemfile

@@ -1,3 +1,6 @@
+#ruby=ruby-2.0.0
+#ruby-gemset=cid
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in cid.gemspec

data/Rakefile

@@ -1 +1,17 @@
 require "bundler/gem_tasks"
+
+$:.unshift File.join( File.dirname(__FILE__), "lib")
+
+require 'rubygems'
+require 'cucumber'
+require 'cucumber/rake/task'
+require 'coveralls/rake/task'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+Coveralls::RakeTask.new
+Cucumber::Rake::Task.new(:features) do |t|
+  t.cucumber_opts = "features --format pretty"
+end
+
+task :default => [:spec, :features, 'coveralls:push']

data/bin/cid

@@ -48,4 +48,22 @@ if ARGV[0] == "validate"
 
   exit code
 
+elsif ARGV[0] == "publish"
+
+  if ARGV[1]
+    source = ARGV[1]
+  else
+    source = "."
+  end
+
+  datapackage = Cid::Datapackage.new(source)
+  datapackage.create
+  datapackage.write
+  if File.exist?("#{source}/.git")
+    datapackage.publish
+    puts "Changes pushed successfully".colorize(:green)
+  else
+    puts "Git repo not found".colorize(:red)
+    exit 1
+  end
 end

data/cid.gemspec

@@ -19,11 +19,22 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "csvlint"
   spec.add_dependency "colorize"
+  spec.add_dependency "json"
+  spec.add_dependency "git"
+  spec.add_dependency "github_api"
+  spec.add_dependency "dotenv"
+  spec.add_dependency "memoist"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-remote"
   spec.add_development_dependency "coveralls"
+  spec.add_development_dependency "cucumber"
+  spec.add_development_dependency "aruba"
+  spec.add_development_dependency "spork"
+  spec.add_development_dependency "vcr"
+  spec.add_development_dependency "webmock"
 
 end

data/features/publish.feature

@@ -0,0 +1,29 @@
+@publish
+Feature: Publish a datapackage
+
+  Scenario: Create datapackage
+    Given I run `cp -r ../../spec/fixtures/valid valid`
+    And I cd to "valid"
+    When I run `cid publish`
+    And I run `cat datapackage.json`
+    Then the output should contain "votes/votes-1.csv"
+
+  Scenario: Create datapackage with new files
+    Given I run `cp -r ../../spec/fixtures/multiple_files multiple_files`
+    And I cd to "multiple_files"
+    When I run `cid publish`
+    And I run `cat datapackage.json`
+    Then the output should contain "votes/votes-1.csv"
+    And the output should contain "votes/votes-2.csv"
+
+  Scenario: Return error if not in a git repo
+    Given I run `cp -r ../../spec/fixtures/multiple_files multiple_files`
+    And I cd to "multiple_files"
+    When I run `cid publish`
+    Then the output should contain "Git repo not found"
+
+  Scenario: Sucessfully push to Github
+    Given I run `git clone https://github.com/theodi/cid-test.git`
+    And I cd to "cid-test"
+    When I successfully run `cid publish`
+    Then the output should contain "Changes pushed successfully"

data/features/step_definitions/cid.rb

@@ -0,0 +1,7 @@
+Given(/^I set up a new git repo$/) do
+  steps %{
+    And I run `git init`
+    And I run `git add .`
+    And I run `git commit -am 'New git repo'`
+  }
+end

data/features/support/env.rb

@@ -0,0 +1,28 @@
+$:.unshift File.join( File.dirname(__FILE__), "..", "..", "lib")
+
+require 'rspec/expectations'
+require 'pry'
+require 'pry-remote'
+require 'spork'
+require 'aruba/cucumber'
+require 'dotenv'
+
+Dotenv.load
+
+World(RSpec::Matchers)
+
+Before do
+  @aruba_timeout_seconds = 10
+end
+
+Before('@validate') do
+  @dirs = ["#{File.dirname(__FILE__)}/../.."]
+end
+
+After('@publish') do
+  `rm -rf tmp/aruba/`
+end
+
+Spork.each_run do
+  require 'cid'
+end

data/features/validate.feature

@@ -0,0 +1,30 @@
+@validate
+Feature: Validate a datapackage
+
+  Scenario: Package with a single CSV
+    When I cd to "spec/fixtures/valid"
+    And I run `cid validate`
+    Then the output should contain "votes/votes-1.csv is VALID"
+
+  Scenario: Package with multiple CSVs
+    When I cd to "spec/fixtures/multiple_files"
+    And I run `cid validate`
+    Then the output should contain "votes/votes-1.csv is VALID"
+    And the output should contain "votes/votes-2.csv is VALID"
+
+  Scenario: Package with multiple folders
+    When I cd to "spec/fixtures/multiple_folders"
+    And I run `cid validate`
+    Then the output should contain "seats/seats-1.csv is VALID"
+    And the output should contain "seats/seats-2.csv is VALID"
+    And the output should contain "votes/votes-1.csv is VALID"
+    And the output should contain "votes/votes-2.csv is VALID"
+
+  Scenario: Package with invalid CSV
+    When I cd to "spec/fixtures/invalid"
+    And I run `cid validate`
+    Then the output should contain "votes/votes-1.csv is INVALID"
+
+  Scenario: Specify directory
+    When I run `cid validate spec/fixtures/valid`
+    Then the output should contain "votes/votes-1.csv is VALID"

data/lib/cid/datapackage.rb

@@ -0,0 +1,57 @@
+require 'csvlint'
+require 'json'
+
+module Cid
+
+  class Datapackage
+
+    attr_accessor :json
+
+    def initialize(path)
+      @path = path
+      begin
+        @datapackage = JSON.parse(File.new("#{@path}/datapackage.json").read)
+      rescue
+        return "No datapackage present!"
+      end
+    end
+
+    def create
+      # Clear out all the resources
+      @datapackage["resources"] = []
+
+      paths = Dir.glob("#{@path}/*/")
+
+      paths.each do |path|
+        schema = JSON.parse(File.new(Dir["#{path}/schema.json"][0]).read)
+
+        Dir["#{path}*.csv"].each do |csv|
+          @datapackage["resources"] << {
+            name: File.basename(csv, ".csv"),
+            path: csv.split("/").last(2).join("/"),
+            format: "csv",
+            mediatype: "text/csv",
+            bytes: File.size(csv),
+            schema: {
+                fields: schema["fields"]
+              }
+          }
+        end
+      end
+
+      @json = @datapackage.to_json
+    end
+
+    def write
+      self.create if @json.nil?
+      File.open("#{@path}/datapackage.json", 'w') { |f| f.write(@json) }
+    end
+
+    def publish
+      git = Cid::Publish.new(@path, ENV['GITHUB_OAUTH_TOKEN'])
+      git.publish
+    end
+
+  end
+
+end

data/lib/cid/helpers/github.rb

@@ -0,0 +1,91 @@
+require 'base64'
+require 'github_api'
+require 'memoist'
+
+module Cid::Helpers::Github
+
+  extend Memoist
+
+  GITHUB_REPO_REGEX = /github.com[:\/]([^\/]*)\/([^\.]*)/
+
+  def github
+    Github.new oauth_token: @oauth_token
+  end
+  memoize :github
+
+  def user
+    match = @git_url.match GITHUB_REPO_REGEX
+    match ? match[1] : nil
+  end
+  memoize :user
+
+  def repo
+    match = @git_url.match GITHUB_REPO_REGEX
+    match ? match[2] : nil
+  end
+  memoize :repo
+
+  def default_branch
+    repository = github.repos.get user, repo
+    repository.default_branch
+  end
+  memoize :default_branch
+
+  def latest_commit(branch_name)
+    branch_data = github.repos.branch user, repo, branch_name
+    branch_data['commit']['sha']
+  end
+  memoize :latest_commit
+
+  def tree(branch)
+    github.git_data.trees.get user, repo, branch
+  end
+  memoize :tree
+
+  def blob_shas(branch, path)
+    tree = tree branch
+    Hash[tree['tree'].select{|x| x['path'] =~ /^#{path}$/ && x['type'] == 'blob'}.map{|x| [x.path, x.sha]}]
+  end
+  memoize :blob_shas
+
+  def blob_content(sha)
+    blob = github.git_data.blobs.get user, repo, sha
+    if blob['encoding'] == 'base64'
+      Base64.decode64(blob['content'])
+    else
+      blob['content']
+    end
+  end
+  memoize :blob_content
+
+
+  def create_blob(content)
+    blob = github.git_data.blobs.create user, repo, "content" => content, "encoding" => "utf-8"
+    blob['sha']
+  end
+
+  def add_blob_to_tree(sha, filename)
+    tree = tree default_branch
+    new_tree = github.git_data.trees.create user, repo, "base_tree" => tree['sha'], "tree" => [
+      "path" => filename,
+      "mode" => "100644",
+      "type" => "blob",
+      "sha" => sha
+    ]
+    new_tree['sha']
+  end
+
+  def commit(sha)
+    parent = latest_commit(default_branch)
+    commit = github.git_data.commits.create user, repo, "message" => "Updated datapackage.json",
+              "parents" => [parent],
+              "tree" => sha
+    commit['sha']
+  end
+
+  def push(sha)
+    branch = github.git_data.references.update user, repo, "heads/#{default_branch}", "sha" => sha
+    branch['ref']
+  end
+
+end

data/lib/cid/publish.rb

@@ -0,0 +1,28 @@
+require 'git'
+
+module Cid
+
+  class Publish
+
+    include Cid::Helpers::Github
+
+    GITHUB_REPO_REGEX = /github.com[:\/]([^\/]*)\/([^\.]*)/
+
+    def initialize(path, oauth_token)
+      g = Git.open(path)
+      @path = path
+      @git_url = g.config["remote.origin.url"]
+      @oauth_token = oauth_token
+    end
+
+    def publish
+      name, content = "datapackage.json", File.read("#{@path}/datapackage.json")
+      blob_sha = create_blob(content)
+      tree_sha = add_blob_to_tree(blob_sha, name)
+      commit_sha = commit(tree_sha)
+      push(commit_sha)
+    end
+
+  end
+
+end

data/lib/cid/validation.rb

@@ -13,7 +13,7 @@ module Cid
         begin
           schema = Csvlint::Schema.load_from_json_table(File.new(Dir["#{path}/schema.json"][0]))
         rescue
-          # Error! No schema in place!
+          schema = nil
         end
 
         Dir["#{path}/*.csv"].each do |csv|

data/lib/cid/version.rb

@@ -1,3 +1,3 @@
 module Cid
-  VERSION = "0.0.2"
+  VERSION = "0.1.0"
 end

data/lib/cid.rb

@@ -1,2 +1,13 @@
+module Cid
+
+  module Helpers
+  end
+
+end
+
+require "cid/helpers/github"
+
 require "cid/version"
 require "cid/validation"
+require "cid/datapackage"
+require "cid/publish"

data/spec/cassettes/Cid_Publish/should_be_able_to_create_a_new_blob.yml

@@ -0,0 +1,81 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: https://api.github.com/repos/theodi/cid-test/git/blobs?access_token=<GITHUB_OAUTH_TOKEN>
+    body:
+      encoding: UTF-8
+      string: '{"content":"new blob content","encoding":"utf-8"}'
+    headers:
+      Accept:
+      - application/vnd.github.v3+json,application/vnd.github.beta+json;q=0.5,application/json;q=0.1
+      Accept-Charset:
+      - utf-8
+      User-Agent:
+      - Github Ruby Gem 0.11.3
+      Content-Type:
+      - application/json
+      Authorization:
+      - Token token="<GITHUB_OAUTH_TOKEN>"
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+  response:
+    status:
+      code: 201
+      message: Created
+    headers:
+      Server:
+      - GitHub.com
+      Date:
+      - Thu, 22 May 2014 14:09:04 GMT
+      Content-Type:
+      - application/json; charset=utf-8
+      Status:
+      - 201 Created
+      X-Ratelimit-Limit:
+      - '5000'
+      X-Ratelimit-Remaining:
+      - '4998'
+      X-Ratelimit-Reset:
+      - '1400771341'
+      Cache-Control:
+      - private, max-age=60, s-maxage=60
+      Etag:
+      - '"3cb71254860221b3fe02e4d6afd3b069"'
+      X-Oauth-Scopes:
+      - gist, repo, user
+      X-Accepted-Oauth-Scopes:
+      - ''
+      Location:
+      - https://api.github.com/repos/theodi/cid-test/git/blobs/28b552e7359c5c3bbe947749aab70d18e3ea554b
+      Vary:
+      - Accept, Authorization, Cookie, X-GitHub-OTP
+      X-Github-Media-Type:
+      - github.v3; format=json
+      X-Xss-Protection:
+      - 1; mode=block
+      X-Frame-Options:
+      - deny
+      Content-Security-Policy:
+      - default-src 'none'
+      Content-Length:
+      - '154'
+      Access-Control-Allow-Credentials:
+      - 'true'
+      Access-Control-Expose-Headers:
+      - ETag, Link, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset,
+        X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval
+      Access-Control-Allow-Origin:
+      - '*'
+      X-Github-Request-Id:
+      - 4F4C59B3:0FE8:383DBFF:537E0500
+      Strict-Transport-Security:
+      - max-age=31536000
+      X-Content-Type-Options:
+      - nosniff
+    body:
+      encoding: UTF-8
+      string: '{"sha":"28b552e7359c5c3bbe947749aab70d18e3ea554b","url":"https://api.github.com/repos/theodi/cid-test/git/blobs/28b552e7359c5c3bbe947749aab70d18e3ea554b"}'
+    http_version: 
+  recorded_at: Thu, 22 May 2014 14:09:09 GMT
+recorded_with: VCR 2.4.0