datapackage 0.1.3 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be13eda8579c72ff6a07c27bfeef9c427ca8b6ae
-  data.tar.gz: cbe87da5508914dda437c41c3fb7143a52623128
+  metadata.gz: 5fc6a10e6daa3511af308ad0bc2d3c8901c13237
+  data.tar.gz: 28cc6fe87eb009d93c31ae5341cc599eefd0c96b
 SHA512:
-  metadata.gz: 2cbeb0ad411624427b34c10a7523052c435e35a2aa198dbef6f1ddb320eca09f13b91556d52a8f497fadda7b870011c6dae5fb4a33f59256253996f76ef53821
-  data.tar.gz: 68dd2c8774787319bf17522c7ce1fa61304947e00a5003de0c370f9e62300c00d10284b6667bfeadf7dc39cbc2d376fe05307b486b09e7467340a1b5660fc481
+  metadata.gz: b59f052947c6a931890c76dc005ce10f2cb1e36a593e6c089c503a838dceb87cd2a0662b93b53e7ed6299b4e285b5ba75dff4a07fbeb3f1f48d0f69525ccc96c
+  data.tar.gz: a1423662c7310e957db54c8c566a8fc044c78489188094c844eae13dee81e9977f1befde61474b869fcd9b3b79cd2458bfb906ae81da59d90140fc8380208952

data/README.md

@@ -6,7 +6,7 @@
 [![SemVer](https://img.shields.io/badge/versions-SemVer-brightgreen.svg)](http://semver.org/)
 [![Gitter](https://img.shields.io/gitter/room/frictionlessdata/chat.svg)](https://gitter.im/frictionlessdata/chat)
 
-A ruby library for working with [Data Packages](http://dataprotocols.org/data-packages/).
+A ruby library for working with [Data Packages](https://specs.frictionlessdata.io/data-package/).
 
 The library is intending to support:
 
@@ -35,29 +35,54 @@ Require the gem, if you need to:
 require 'datapackage'
 ```
 
-Parsing a Data Package from a remote location:
+Parsing a data package descriptor from a remote location:
 
 ```ruby
-package = DataPackage::Package.new( "http://example.org/datasets/a" )
+package = DataPackage::Package.new( "http://example.org/datasets/a/datapackage.json" )
 ```
 
-This assumes that `http://example.org/datasets/a/datapackage.json` exists, or specifically load a JSON file:
+This assumes that `http://example.org/datasets/a/datapackage.json` exists.
+Similarly you can load a package descriptor from a local JSON file.
 
 ```ruby
-package = DataPackage::Package.new( "http://example.org/datasets/a/datapackage.json" )
+package = DataPackage::Package.new( "/my/data/package/datapackage.json" )
 ```
 
-Similarly you can load a package from a local JSON file, or specify a directory:
+The data package descriptor
+i.e. `datapackage.json` file, is expected to be at the _root_ directory
+of the data package and the `path` attribute of the package's `resources` will be resolved
+relative to it.
+
+You can also load a data package descriptor directly from a Hash:
 
 ```ruby
-package = DataPackage::Package.new( "/my/data/package" )
-package = DataPackage::Package.new( "/my/data/package/datapackage.json" )
+ descriptor = {
+  'resources'=> [
+    {
+      'name'=> 'example',
+      'profile'=> 'tabular-data-resource',
+      'data'=> [
+        ['height', 'age', 'name'],
+        ['180', '18', 'Tony'],
+        ['192', '32', 'Jacob'],
+      ],
+      'schema'=>  {
+        'fields'=> [
+          {'name'=> 'height', 'type'=> 'integer'},
+          {'name'=> 'age', 'type'=> 'integer'},
+          {'name'=> 'name', 'type'=> 'string'},
+        ],
+      }
+    }
+  ]
+}
+
+package = DataPackage::Package.new(descriptor)
 ```
 
 There are a set of helper methods for accessing data from the package, e.g:
 
 ```ruby
-package = DataPackage::Package.new( "/my/data/package" )
 package.name
 package.title
 package.description
@@ -65,93 +90,154 @@ package.homepage
 package.license
 ```
 
-## Reading a Data Package and its resources
+## Reading Data Resources
 
-```ruby
-require 'datapackage'
+A data package must contain an array of [Data Resources](https://specs.frictionlessdata.io/data-resource).
+You can access the resources in your Data Package either by their name or by their index in the `resources` array:
 
-dp = DataPackage::Package.new('http://data.okfn.org/data/core/gdp/datapackage.json')
+```ruby
+first_resource = package.resources[0]
+first_resource = package.get_resource('example')
 
-data = CSV.parse(dp.resources[0].data, headers: true)
-brazil_gdp = data.select { |r| r["Country Code"] == "BRA" }.
-                  map { |row| { year: Integer(row["Year"]), value: Float(row['Value']) } }
+# Get info about the data source of this resource
+first_resource.source_type
+first_resource.source
+```
 
-max_gdp = brazil_gdp.max_by { |r| r[:value] }
-min_gdp = brazil_gdp.min_by { |r| r[:value] }
+You can then read the source depending on its `source_type`: `inline`, `remote` or `local`.
 
-percentual_increase = (max_gdp[:value] / min_gdp[:value]).round(2)
-max_gdp_val = max_gdp[:value].to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+If a resource complies with the [Tabular Data Resource spec](https://specs.frictionlessdata.io/tabular-data-resource/) or uses the
+`tabular-data-resource` [profile](#profiles) you can make a [TableSchema::Table](https://github.com/frictionlessdata/tableschema-rb) for it:
 
-msg =  "The highest Brazilian GDP occured in #{max_gdp[:year]}, when it peaked at US$ " +
-"#{max_gdp_val}. This was #{percentual_increase}% more than its minumum GDP " +
-"in #{min_gdp[:year]}"
+```ruby
+package.resources[0].tabular?
+table = package.resources[0].table
 
-print msg
+# Read the entire table at once
+data = table.read
 
-# The highest Brazilian GDP occured in 2011, when it peaked at US$ 2,615,189,973,181. This was 172.44% more than its minimum GDP in 1960.
+# Or iterate through it
+data = table.iter {|row| print row}
 ```
 
+See [TableSchema](https://github.com/frictionlessdata/tableschema-rb) documentation for other things you can do with tabular resource.
+
 ## Creating a Data Package
 
 ```ruby
 package = DataPackage::Package.new
 
+# Add package properties
 package.name = 'my_sleep_duration'
-package.resources =  [
-  {'name': 'data'}
-]
-
-resource = package.resources[0]
-resource.descriptor['data'] = [
-  7, 8, 5, 6, 9, 7, 8
-]
 
-File.open('datapackage.json', 'w') do |f|
-  f.write(package.to_json)
-end
-
-# {"name": "my_sleep_duration", "resources": [{"name": "data", "data": [7, 8, 5, 6, 9, 7, 8]}]}
+# Add a resource
+package.add_resource(
+  {
+    'name'=> 'sleep_durations_this_week',
+    'data'=> [7, 8, 5, 6, 9, 7, 8],
+  }
+)
 ```
 
-## Validating a Data Package
+If the resource is valid it will be added to the `resources` array of the Data Package;
+if it's invalid it will not be added and you should try creating and [validating](#validating-a-resource) your resource to see why it fails.
 
 ```ruby
-package = DataPackage::Package.new('http://data.okfn.org/data/core/gdp/datapackage.json')
+# Update a resource
+my_resource = package.get_resource('sleep_durations_this_week')
+my_resource['schema'] = {
+  'fields'=> [
+    {'name'=> 'number_hours', 'type'=> 'integer'},
+  ]
+}
+
+# Save the Data Package descriptor to the target file
+package.save('datapackage.json')
 
-package.valid?
-#=> true
-package.errors
-#=> [] # An array of errors
+# Remove a resource
+package.remove_resource('sleep_durations_this_week')
 ```
 
-## Using a different schema
+## Profiles
+
+Data Package and Data Resource descriptors can be validated against  [JSON schemas](https://tools.ietf.org/html/draft-zyp-json-schema-04) that we call `profiles`.
+
+By default, this gem uses the standard [Data Package profile](http://specs.frictionlessdata.io/schemas/data-package.json) and [Data Resource profile](http://specs.frictionlessdata.io/schemas/data-resource.json) but alternative profiles are available for both.
 
-By default, the gem uses the standard [Data Package Schema](http://specs.frictionlessdata.io/data-packages/), but alternative schemas are available.
+According to the [specs](https://specs.frictionlessdata.io/profiles/) the value of
+the `profile` property can be either a URL or an indentifier from [the registry](https://specs.frictionlessdata.io/schemas/registry.json).
 
-### Schemas in the local cache
+### Profiles in the local cache
 
-The gem comes with schemas for the standard Data Package Schema, as well as the [Tabular Data Package Schema](http://specs.frictionlessdata.io/tabular-data-package/), and the [Fiscal Data Package Schema](http://fiscal.dataprotocols.org/spec/). These can be referred to via an identifier, expressed as a symbol.
+The profiles from the registry come bundled with the gem. You can reference them in your Data Package descriptor by their identifier in [the registry](https://specs.frictionlessdata.io/schemas/registry.json):
+
+- `data-package` the default profile for a [Data Package](https://specs.frictionlessdata.io/data-package/)
+- `data-resource` the default profile for a [Data Resource](https://specs.frictionlessdata.io/data-resource)
+- `tabular-data-package` for a [Tabular Data Package](http://specs.frictionlessdata.io/tabular-data-package/)
+- `tabular-data-resource` for a [Tabular Data Resource](https://specs.frictionlessdata.io/tabular-data-resource/)
+- `fiscal-data-package` for a [Fiscal Data Package](http://fiscal.dataprotocols.org/spec/)
 
 ```ruby
-package = DataPackage::Package.new(nil, :tabular) # Or :fiscal
+{
+  "profile": "tabular-data-package"
+}
 ```
 
-### Schemas from elsewhere
+### Profiles from elsewhere
 
-If you have a schema stored in an alternative registry, you can pass a `registry_url` option to the initializer.
+If you have a custom profile schema you can reference it by its URL:
 
 ```ruby
-package = DataPackage::Package.new(nil, :identifier, {registry_url: 'http://example.org/my-registry.csv'} )
+{
+  "profile": "https://specs.frictionlessdata.io/schemas/tabular-data-package.json"
+}
 ```
 
+## Validation
+
+Data Resources and Data Packages are validated against their profiles to ensure they respect the expected structure.
+
+### Validating a Resource
+
+```ruby
+descriptor = {
+  'name'=> 'incorrect name',
+  'path'=> 'https://cdn.rawgit.com/frictionlessdata/datapackage-rb/master/spec/fixtures/test-pkg/test.csv',
+}
+resource = DataPackage::Resource.new(descriptor, base_path='')
+
+# Returns true if resource is valid, false otherwise
+resource.valid?
+
+# Returns true or raises DataPackage::ValidationError
+resource.validate
+
+# Iterate through validation errors
+resource.iter_errors{ |err| p err}
+```
+
+### Validating a Package
+
+The same methods used to check the validity of a Resource - `valid?`, `validate` and `iter_errors`- are also available for a Package.
+The difference is that after a Package descriptor is validated against its `profile`, each of its `resources` are also validated against their `profile`.
+
+In order for a Package to be valid all its Resources have to be valid.
+
 ## Developer notes
 
 These notes are intended to help people that want to contribute to this package itself. If you just want to use it, you can safely ignore them.
 
+After checking out the repo, run `bundle` to install dependencies. Then, run `rake spec` to run the tests.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`,
+which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ### Updating the local schemas cache
 
-We cache the schemas from https://github.com/dataprotocols/schemas using git-subtree. To update it, use:
+We cache the local schemas from https://specs.frictionlessdata.io/schemas/registry.json.
+The local schemas should be kept up to date with the remote ones using:
 
 ```
-  git subtree pull --prefix datapackage/schemas https://github.com/dataprotocols/schemas.git master --squash
+rake update_profiles
 ```

data/bin/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "datapackage"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require "pry"
+Pry.start

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapackage
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.1
 platform: ruby
 authors:
 - Leigh Dodds
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-19 00:00:00.000000000 Z
+date: 2017-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -40,20 +40,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rest-client
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -68,20 +54,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rack
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.6.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.6.4
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -111,7 +83,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: jsontableschema
+  name: tableschema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -213,33 +185,14 @@ email:
 - ops@theodi.org
 executables:
 - datapackage
+- console
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE.md
 - README.md
+- bin/console
 - bin/datapackage
-- datapackage/schemas/LICENSE.md
-- datapackage/schemas/README.md
-- datapackage/schemas/csv-dialect-description-format.json
-- datapackage/schemas/data-package.json
-- datapackage/schemas/definitions.json
-- datapackage/schemas/fiscal-data-package.json
-- datapackage/schemas/fiscal-data-package.jsonld
-- datapackage/schemas/index.html
-- datapackage/schemas/json-table-schema.json
-- datapackage/schemas/registry.csv
-- datapackage/schemas/tabular-data-package.json
-- datapackage/schemas/tests/__init__.py
-- datapackage/schemas/tests/test_registry.py
-- datapackage/schemas/tests/test_schemas.py
-- lib/datapackage.rb
-- lib/datapackage/exceptions.rb
-- lib/datapackage/package.rb
-- lib/datapackage/registry.rb
-- lib/datapackage/resource.rb
-- lib/datapackage/schema.rb
-- lib/datapackage/version.rb
 homepage: http://github.com/theodi/datapackage.rb
 licenses:
 - MIT

data/datapackage/schemas/LICENSE.md

@@ -1,24 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <http://unlicense.org>

data/datapackage/schemas/README.md

@@ -1,33 +0,0 @@
-# JSON Schemas for Data Protocol Formats
-
-[![Build Status](http://travis-ci.org/dataprotocols/schemas.svg?branch=master)](http://travis-ci.org/dataprotocols/schemas)
-
-JSON Schemas, and a registry, for the Data Package family of specifications. Read more about Data Packages at [Data Protocols](http://dataprotocols.org/).
-
-The schemas are implemented using [JSON Schema](http://json-schema.org/), a specification which provides a simple declarative format for describing the structure of JSON documents.
-
-The registry is implemented as simple CSV file, and there are libraries in [Javascript](http://github.com/okfn/datapackage-registry-js) and [Python](http://github.com/okfn/datapackage-registry-py) that work with the registry directly.
-
-## The schemas
-
-Here you'll find schemas for [Data Package](http://dataprotocols.org/data-packages/), various Data Package Profiles, [JSON Table Schemas](http://dataprotocols.org/json-table-schema/), [CSV Dialect Description Format](http://dataprotocols.org/csv-dialect/) and more.
-
-Note that some of the schemas also feature information for [json-editor](http://github.com/jdorn/json-editor) - useful for building web forms and other UI components dynamically from a schema. We use this extensively in [DataPackagist](http://github.com/okfn/datapackagist) to build UIs for creating Data Packages.
-
-## The registry
-
-The registry enables consumers to get access to schemas and documentation for the family of Data Package specifications, and related specifications like JSON Table Schema and CSV Dialect Description Format. See [Data Protocols](http://dataprotocols.org/) for more information.
-
-### Contributing
-
-Yes we welcome and encourage additions to the registry! Any spec that is added must meet the following criteria:
-
-* Be related to the Data Packages family of specifications.
-* Have a publicly -accessible web page describing the specification.
-* Have a JSON Schema file that describes the specification.
-
-See the existing entries in the registry, and then take the following steps to add a new entry:
-
-1. Make a new pull request called `registry/{NAME_OF_SPECIFICATION}`
-2. The pull request features a JSON Schema file for the new specification, and adds the spec to `registry.csv`
-3. Write a brief description of the spec as part of the pull request.

data/datapackage/schemas/csv-dialect-description-format.json

@@ -1,30 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "CSVDDF",
-  "description": "JSON Schema for validating CSVDDF dialect structures",
-  "type": "object",
-  "properties": {
-    "delimiter": {
-      "type": "string"
-    },
-    "doublequote": {
-      "type": "boolean"
-    },
-    "lineterminator": {
-      "type": "string"
-    },
-    "quotechar": {
-      "type": "string"
-    },
-    "skipinitialspace": {
-      "type": "boolean"
-    }
-  },
-  "required": [
-    "delimiter",
-    "doublequote",
-    "lineterminator",
-    "quotechar",
-    "skipinitialspace"
-  ]
-}

data/datapackage/schemas/data-package.json

@@ -1,146 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Data Package",
-  "description": "Data Package is a simple specification for data access and delivery.",
-  "type": "object",
-  "required": [ "name", "resources" ],
-  "properties": {
-    "name": {
-      "$ref": "definitions.json#/define/name",
-      "propertyOrder": 10
-    },
-    "title": {
-      "$ref": "definitions.json#/define/title",
-      "propertyOrder": 20
-    },
-    "description": {
-      "$ref": "definitions.json#/define/description",
-      "format": "textarea",
-      "propertyOrder": 30
-    },
-    "homepage": {
-      "$ref": "definitions.json#/define/homepage",
-      "propertyOrder": 40
-    },
-    "version": {
-      "$ref": "definitions.json#/define/version",
-      "propertyOrder": 50
-    },
-    "license": {
-      "$ref": "definitions.json#/define/license",
-      "propertyOrder": 60
-    },
-    "author": {
-      "$ref": "definitions.json#/define/author",
-      "propertyOrder": 70
-    },
-    "contributors": {
-      "$ref": "definitions.json#/define/contributors",
-      "propertyOrder": 80,
-      "options": { "hidden": true }
-    },
-    "resources": {
-      "title": "Resources",
-      "description": "The data resources that this package describes.",
-      "type": "array",
-      "propertyOrder": 90,
-      "minItems": 0,
-      "items": {
-        "type": "object",
-        "properties": {
-          "name": {
-            "$ref": "definitions.json#/define/name",
-            "propertyOrder": 10
-          },
-          "title": {
-            "$ref": "definitions.json#/define/title",
-            "propertyOrder": 20
-          },
-          "description": {
-            "$ref": "definitions.json#/define/description",
-            "propertyOrder": 30,
-            "format": "textarea"
-          },
-          "schema": {
-            "$ref": "definitions.json#/define/schema",
-            "propertyOrder": 40
-          },
-          "url": {
-            "$ref": "definitions.json#/define/url",
-            "propertyOrder": 50
-          },
-          "path": {
-            "$ref": "definitions.json#/define/path",
-            "propertyOrder": 60
-          },
-          "data": {
-            "$ref": "definitions.json#/define/data",
-            "propertyOrder": 70
-          },
-          "format": {
-            "$ref": "definitions.json#/define/format",
-            "propertyOrder": 80
-          },
-          "mediatype": {
-            "$ref": "definitions.json#/define/mediatype",
-            "propertyOrder": 90
-          },
-          "encoding": {
-            "$ref": "definitions.json#/define/encoding",
-            "propertyOrder": 100
-          },
-          "bytes": {
-            "$ref": "definitions.json#/define/bytes",
-            "propertyOrder": 110,
-            "options": { "hidden": true }
-          },
-          "hash": {
-            "$ref": "definitions.json#/define/hash",
-            "propertyOrder": 120,
-            "options": { "hidden": true }
-          },
-          "dialect": {
-            "$ref": "definitions.json#/define/dialect",
-            "propertyOrder": 130,
-            "options": { "hidden": true }
-          },
-          "sources": {
-            "$ref": "definitions.json#/define/sources",
-            "propertyOrder": 140,
-            "options": { "hidden": true }
-          },
-          "license": {
-            "$ref": "definitions.json#/define/license",
-            "description": "The license under which the resource is published.",
-            "propertyOrder": 150,
-            "options": { "hidden": true }
-          }
-        },
-        "anyOf": [
-          { "title": "url required", "required": ["url"] },
-          { "title": "path required", "required": ["path"] },
-          { "title": "data required", "required": ["data"] }
-        ]
-      }
-    },
-    "keywords": {
-      "$ref": "definitions.json#/define/keywords",
-      "propertyOrder": 100
-    },
-    "sources": {
-      "$ref": "definitions.json#/define/sources",
-      "propertyOrder": 110,
-      "options": { "hidden": true }
-    },
-    "image": {
-      "$ref": "definitions.json#/define/image",
-      "propertyOrder": 120,
-      "options": { "hidden": true }
-    },
-    "dataDependencies": {
-      "$ref": "definitions.json#/define/dataDependencies",
-      "propertyOrder": 140,
-      "options": { "hidden": true }
-    }
-  }
-}