dbd 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rvmrc

@@ -0,0 +1 @@
+rvm --create use 2.0.0@dbd

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+bundler_args: --without debug
+script: "bundle exec rspec spec"
+rvm:
+  - 2.0.0
+  - 1.9.3
+  - 1.9.2
+  - jruby-19mode
+  - ruby-head
+  - jruby-head

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dbd.gemspec
+gemspec
+
+#gem 'rdf', '1.1.0',
+#    :git => 'https://github.com/ruby-rdf/rdf',
+#    :branch => '1.1'

data/Guardfile

@@ -0,0 +1,7 @@
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^spec/factories})    { "spec" }
+  watch(%r{^lib/dbd/helpers})       { "spec" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Peter Vandenabeele
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,97 @@
+# Dbd: A data store that (almost) never forgets
+
+This is facts based data store, inspired by [RDF] concepts, but adding a log based structure and fine-grained provenance.
+
+* [Why?][Rationale]
+* <http://github.com/petervandenabeele/dbd>
+
+[![Gem Version](https://badge.fury.io/rb/dbd.png)](http://badge.fury.io/rb/dbd)
+[![Build Status](https://travis-ci.org/petervandenabeele/dbd.png?branch=master)](http://travis-ci.org/petervandenabeele/dbd)
+
+## Features
+
+* Facts are immutable and ordered (log structured "fact stream")
+  * "Big Data" mantra: base facts are immutable (store now, analyse later)
+  * only 1 backup file needed: the fact stream contains the full history
+  * synchronisation between remote sources is cheap (easy caching)
+  * 1 data source has _all_ my data : never more loose stuff :-)
+  * facts can be invalidated (and replaced) later on
+* Privacy
+  * a "hard delete" is possible: all downstream readers of the fact stream
+    must remove this fact and replace the back-up
+  * since one single back-up file suffices, replacing the *single* back-up  
+    file will actually remove the hard deleted fact(s) for good
+* Fine grained Provenance
+  * Each base Fact points to a ProvenanceResource, so separate provenance  
+    possible for different facts about 1 resource
+  * can keep the original_source reference, creator, date, …
+  * can have a context that allows filtering data (e.g. private, professional, …)
+  * separate encryption schemes per context are possible
+  * ProvenanceResource is flexible, since built itself from Facts
+* Schemaless
+  * uses the [RDF] (subject, predicate, object) concepts
+  * predicates, types can be defined in an ontology for declaring meaning
+* Graph based
+  * the object of each Fact can be another Resource
+  * aimed at exporting to a graph database (e.g. Neo4j) for analysis
+
+
+## License
+
+Open Source [MIT]
+
+## Installation
+
+    $ gem install dbd      # Ruby 1.9.2, 1.9.3, 2.0.x, jruby (see .travis.yml)
+
+## Examples
+
+    require 'dbd'
+
+    provenance = Dbd::ProvenanceResource.new
+
+    # PREFIX provenance: <https://data.vandenabeele.com/ontologies/provenance#>
+    # PREFIX dcterms: <http://purl.org/dc/terms/>
+    fact_context_public = Dbd::ProvenanceFact.new(predicate: "provenance:context", object: "public")
+    fact_creator_peter_v = Dbd::ProvenanceFact.new(predicate: "dcterms:creator", object: "@peter_v")
+    fact_created_at_now = Dbd::ProvenanceFact.new(predicate: "provenance:created_at", object: Time.now.utc)
+    provenance << fact_context_public
+    provenance << fact_creator_peter_v
+    provenance << fact_created_at_now
+
+    nobel_peace_2012 = Dbd::Resource.new(provenance_subject: provenance.subject)
+
+    # PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+    # PREFIX base: <https://data.vandenabeele.com/ontologies/base#>
+    fact_nobel_peace_2012 = Dbd::Fact.new(predicate: "base:nobelPeacePriceWinner", object: "2012")
+    fact_EU_label = Dbd::Fact.new(predicate: "rdfs:label", object: "EU") #  this will use some RDF predicates in future
+    fact_EU_comment = Dbd::Fact.new(predicate: "rdfs:comment", object: "European Union")
+    fact_EU_story = Dbd::Fact.new(predicate: "base:story", object: "A long period of peace,\n that is a \"bliss\".")
+    nobel_peace_2012 << fact_nobel_peace_2012
+    nobel_peace_2012 << fact_EU_label
+    nobel_peace_2012 << fact_EU_comment
+    nobel_peace_2012 << fact_EU_story
+
+    graph = Dbd::Graph.new
+
+    provenance.each {|provenance_fact| graph << provenance_fact}
+    nobel_peace_2012.each {|fact| graph << fact}
+
+    puts graph.to_CSV
+
+results in
+
+
+    $ ruby test.rb
+    "611dbc31-6961-4a86-9259-4a2700add783","2013-05-12 21:50:19 UTC","","98b7bb17-9921-4d52-a08a-39667c2abb4c","provenance:context","public"
+    "79e9c0e7-b6fd-4735-817c-8c21c97c9575","2013-05-12 21:50:19 UTC","","98b7bb17-9921-4d52-a08a-39667c2abb4c","dcterms:creator","@peter_v"
+    "7d143a50-8a63-4637-8ab8-c2aa7fc6e12e","2013-05-12 21:50:19 UTC","","98b7bb17-9921-4d52-a08a-39667c2abb4c","provenance:created_at","2013-05-12 21:50:19 UTC"
+    "fd121b00-7934-4e22-81c8-8e810760c686","2013-05-12 21:50:19 UTC","98b7bb17-9921-4d52-a08a-39667c2abb4c","477a2e10-5e34-434d-8fc1-969277f61f9f","base:nobelPeacePriceWinner","2012"
+    "2d852fe1-911f-497d-9485-6c24a6000fbb","2013-05-12 21:50:19 UTC","98b7bb17-9921-4d52-a08a-39667c2abb4c","477a2e10-5e34-434d-8fc1-969277f61f9f","rdfs:label","EU"
+    "ab00b092-65a3-47c0-b10b-837cb0a5ad81","2013-05-12 21:50:19 UTC","98b7bb17-9921-4d52-a08a-39667c2abb4c","477a2e10-5e34-434d-8fc1-969277f61f9f","rdfs:comment","European Union"
+    "a8d6b34b-6e02-4a5e-8529-4785f090866a","2013-05-12 21:50:19 UTC","98b7bb17-9921-4d52-a08a-39667c2abb4c","477a2e10-5e34-434d-8fc1-969277f61f9f","base:story","A long period of peace,
+     that is a ""bliss""."
+
+[RDF]:              http://www.w3.org/RDF/
+[Rationale]:        http://github.com/petervandenabeele/dbd/blob/master/docs/rationale.md
+[MIT]:              https://github.com/petervandenabeele/dbd/blob/master/LICENSE.txt

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/dbd.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dbd/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dbd"
+  spec.version       = Dbd::VERSION
+  spec.authors       = ["Peter Vandenabeele"]
+  spec.email         = ["peter@vandenabeele.com"]
+  spec.description   = %q{A data store that (almost) never forgets}
+  spec.summary       = %q{A data store that (almost) never forgets}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', '> 1.2.4'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'rb-fsevent', '~> 0.9'
+  spec.add_development_dependency 'terminal-notifier-guard'
+  spec.add_development_dependency 'yard'
+  spec.add_runtime_dependency 'neography'
+  spec.add_runtime_dependency 'rdf', '~> 1.0.6'
+  spec.add_runtime_dependency 'ruby_peter_v'
+end

data/docs/rationale.md

@@ -0,0 +1,17 @@
+# A data store that (almost) never forgets
+
+I am tired of keeping data in many different data stores for different purposes, different contexts and losing long term data (old phone numbers, old contacts, meeting notes, etc. etc.). I want to add new data from personal and online sources and then combine, query, reason on all that data in different contexts (private, professional, for a certain customer, ...).
+
+For _all_ facts in the data store, I want to know the "original source" of the data (a very fine grained provenance), who added it, when, etc., so I can check back the sources, filter on the source origin to query and reason only in a certain context, and also use it as a basis for a soft or hard delete of certain data. Typical provenance attributes can be: *context*, *original_source*, *license*, *created_by*, *created_at*, *valid_from*, *valid_until*, etc. An experimental version of a fine grained provenance is at https://data.vandenabeele.com/ontologies/.
+
+One feature of the "context" in the provenance is that a "partial export" is possible of data in a certain context (only private, only professional, only for a certain customer etc.). To enforce the separation between the contexts, an advanaced feature is to have separate encryption schemes or keys for different context (initially an overall external encryption is possible, e.g. by mounting the back-end store on encrypted partitions and encrypting the separate "fact stream" files)
+
+To allow easy (incremental) backup, synchronization, distributed processing, caching, the low level "fact stream" must be of a "logging" type and the operations must be idempotent. I see this as the fundamental aspect of "Big Data". Older facts are never removed, never modified, although they could be "invalidated" by newer facts (and then possibly replaced on a logical level by new facts). An other advantage of "invalidation" at a certain time stamp, is that it is possible to go back to the logical state at a certain historical moment.
+
+One exception is the **"human right to be forgotten"** to which I fully agree. So the possibility for a "hard delete" on certain older facts in the data store is a requirement, implemented in such a way that the resulting data graph after the new "hard delete" of older facts is always valid and exactly equivalent to the "soft delete" case. Because all other data (except "hard deleted" data) is always retained, a newer export will function as a _complete_ back-up, so this can replace the older back-ups and so remove the "hard deleted" entries from all back-ups (to fully implement the "right to be forgotten", also in the back-ups).
+
+The logical structure of the data must be defined by a descriptive ontology, so no hard programming is needed to set-up or modify a schema.  For the provenance (meta) level, the facts of a provenance entry are also stored in the same fact stream. A specific requirement is that all provenance facts for a provenance subject must be in the fact stream, before any fact uses this provenance subject (the reason is that when a fact is processed, to be loaded e.g. in a back-end for analytics, the full provenance for the fact must be known).
+
+Based on the ontology of the data, clients can then query this data store and build automated interfaces for inserting and viewing the logical data graph.
+
+(written in Breskens, on 6 April 2013, after a good day of sailing on the Westerschelde)

data/docs/stories/001_create_a_fact.txt

@@ -0,0 +1,15 @@
+001_store_a_fact
+
+As a client
+I can create a data_fact
+
+A data_fact has:
+* a very fine grained date stamp
+  (creating a de facto order)
+* a guaranteed unique UUID
+* a fact_origin UUID
+
+* a subject ID (a UUID)
+* a property (in a correct Semantic format
+  as used in ontology)
+* an object of format String

data/docs/stories/002_create_a_facts_collection.txt

@@ -0,0 +1,14 @@
+002_create_a_facts_collection
+
+As a client
+I can create a collection of facts
+
+A facts_collection has:
+* an ordered list of facts
+* ordered by time_stamp
+* a << function that adds one fact
+* the enumerable functions allow to loop over the facts
+* other array functions (e.g. []) do not work (no random access)
+* the << function will validate that the time_stamp is strictly larger than the most recent time_stamp in the collection
+* there is no way (yet) to add "older" or "same date" facts
+  (e.g. from a merge) in a facts_collection

data/docs/stories/003_create_a_fact_origin.txt

@@ -0,0 +1,15 @@
+003_create_a_fact_origin.txt
+
+As a client
+I can create a fact origin
+
+A fact_origin has:
+* id
+* context
+* original_source
+* created_by
+* created_at
+* entered_by
+* entered_at
+* valid_from
+* valid_until

data/docs/stories/004_create_fact_origins_collection.txt

@@ -0,0 +1,8 @@
+004_create_fact_origins_collection
+
+As a client
+I can add and retrieve fact_origins to and from a collection
+
+* I create a fact_origin
+* I add the fact_origins to the facts_collection
+* I can efficiently (O(1)) find a fact_origin by id

data/docs/stories/005_CSV_export_the_graph.txt

@@ -0,0 +1,18 @@
+005_CSV_export_the_graph
+
+As a client
+I can ask a CSV export of the graph
+
+* a graph is built now as an (ordered) set of
+  fact_origin_collections and fact_collections
+  (currently limited to 1 fact_origin_collection
+   and 1 fact_collection)
+
+* use UTF-8
+* use comma's (tabs are too hard to confuse with spaces)
+* use "" around strings
+* use newlines at end of a row
+* allow newlines in the strings inbetween ""
+* try to use default Ruby CSV library
+
+* facts are ordered by time_stamp

data/docs/stories/006_refactor_fact_origin_to_provenance_fact.txt

@@ -0,0 +1,20 @@
+006_refactor_fact_origin_to_provenance_fact
+
+As a client
+I can use a ProvenanceFact
+
+* a ProvenanceFact is a subclass of a Fact
+  with additional features:
+  * it's time_stamp must be earlier than any Fact that
+    refers to it through it's provenance_fact_subject.
+    (otherwise said: the collection of ProvenanceFacts
+     about a subject must be complete before it is used;
+     no provenance_fact may be added about a fact that
+     already in the fact stream)
+* a graph is represented internally as a single
+  fact_collection
+* for the provenance_facts, efficient access to all
+  provenance_facts by_subject is possible with a
+  ruby Hash (internally coded in O(1) in C)
+
+* Facts are ordered by time_stamp

data/docs/stories/007_rename_property_to_predicate.txt

@@ -0,0 +1,6 @@
+007_rename_property_to_predicate
+
+As a client
+I can use a Fact with standard terms "subject, predicate, object"
+
+* refactor property into predicate

data/docs/stories/008_testing_different_ruby_versions.txt

@@ -0,0 +1,7 @@
+008_testing_different_ruby_versions
+
+As a client
+I can use the library on Ruby 1.9.2 , 1.9.3 , 2.0.0, JRuby
+
+* test once on these versions
+* find a way for automated testing (travis ?)

data/docs/stories/009_build_and_store_resources_with_provenance.txt

@@ -0,0 +1,38 @@
+009_build_and_store_resources_with_provenance
+
+As a client
+I can build and store a group of resources with provenance
+
+* rename ArrayCollection to OrderedSetCollection
+
+* clean-up the dependency on external UUID gem
+
+* make a FactsBySubject model
+  * add subject and facts related methods
+
+* make a FactsWithProvenance model
+  * add public build methods on it
+
+* add validations on Fact::Collection << for Fact#complete?
+
+* rename FactsBySubject to Resource
+
+* rename FactsWithProvenance to ResourcesWithProvenance
+
+* on Resource, enforce the presence of a subject and set it on the facts
+
+* on Resource, enforce the presence of a provenance_subject and set it on the facts
+
+* on ProvenanceResource, enforce the presence of a subject and set it on the facts
+
+* rename provenance_fact_subject to provenance_subject
+
+* change arguments for (Provenance)Resource to an options hash
+
+* remove the ResourcesWithProvenance
+
+* change arguments for (Provenance)Fact to an options hash
+
+* add a store method on Graph
+  * that will store a (Provenance)Resource instance
+  * this will now set the time_stamps (enforcing the strictly monotic order)

data/docs/stories/010_provenance_fact_properties_from_provenance_ontology.txt

@@ -0,0 +1,10 @@
+010_provenance_fact_predicates_from_provenance_ontology
+
+As a client
+I can read the predicates for a provenance_fact from provenance ontology
+
+* make an ontology in a separate gem
+* import that gem
+* use this ontology to build a validated Provenance Resource
+* use this validate Provenance Resource instance to better buildi
+  a ProvenanceResource

data/docs/test.rb

@@ -0,0 +1,32 @@
+require 'dbd'
+
+provenance = Dbd::ProvenanceResource.new
+
+# PREFIX provenance: <https://data.vandenabeele.com/ontologies/provenance#>
+# PREFIX dcterms: <http://purl.org/dc/terms/>
+fact_context_public = Dbd::ProvenanceFact.new(predicate: "provenance:context", object: "public")
+fact_creator_peter_v = Dbd::ProvenanceFact.new(predicate: "dcterms:creator", object: "@peter_v")
+fact_created_at_now = Dbd::ProvenanceFact.new(predicate: "provenance:created_at", object: Time.now.utc)
+provenance << fact_context_public
+provenance << fact_creator_peter_v
+provenance << fact_created_at_now
+
+nobel_peace_2012 = Dbd::Resource.new(provenance_subject: provenance.subject)
+
+# PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+# PREFIX base: <https://data.vandenabeele.com/ontologies/base#>
+fact_nobel_peace_2012 = Dbd::Fact.new(predicate: "base:nobelPeacePriceWinner", object: "2012")
+fact_EU_label = Dbd::Fact.new(predicate: "rdfs:label", object: "EU") #  this will use some RDF predicates in future
+fact_EU_comment = Dbd::Fact.new(predicate: "rdfs:comment", object: "European Union")
+fact_EU_story = Dbd::Fact.new(predicate: "base:story", object: "A long period of peace,\n that is a \"bliss\".")
+nobel_peace_2012 << fact_nobel_peace_2012
+nobel_peace_2012 << fact_EU_label
+nobel_peace_2012 << fact_EU_comment
+nobel_peace_2012 << fact_EU_story
+
+graph = Dbd::Graph.new
+
+provenance.each {|provenance_fact| graph << provenance_fact}
+nobel_peace_2012.each {|fact| graph << fact}
+
+puts graph.to_CSV

data/lib/dbd.rb

@@ -0,0 +1,13 @@
+require 'ruby_peter_v'
+
+require 'dbd/version'
+
+require 'dbd/errors'
+require 'dbd/fact'
+require 'dbd/provenance_fact'
+require 'dbd/resource'
+require 'dbd/provenance_resource'
+require 'dbd/graph'
+
+require 'dbd/rdf'
+require 'dbd/repo'