ditto 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. data/bin/ditto +1 -1
  2. data/features/create_entities.feature +1 -0
  3. data/features/load_entity.feature +1 -0
  4. data/features/store_entity.feature +5 -4
  5. data/lib/ditto.rb +1 -0
  6. data/lib/ditto/dsl.rb +14 -9
  7. data/lib/ditto/entity.rb +118 -106
  8. data/lib/ditto/error.rb +19 -0
  9. data/lib/ditto/options.rb +45 -13
  10. data/lib/ditto/runner.rb +37 -56
  11. data/lib/ditto/version.rb +1 -1
  12. data/spec/entity_spec.rb +83 -59
  13. data/spec/fixtures/badmap.ditto +18 -0
  14. data/spec/fixtures/badver.ditto +17 -0
  15. data/spec/fixtures/bigcircle.ditto +17 -0
  16. data/spec/fixtures/circle.ditto +16 -0
  17. data/spec/fixtures/continent.ditto +16 -0
  18. data/spec/fixtures/country.ditto +16 -0
  19. data/spec/fixtures/currency.ditto +30 -0
  20. data/spec/fixtures/currency_group.ditto +17 -0
  21. data/spec/fixtures/dot.ditto +17 -0
  22. data/spec/fixtures/dupversion.ditto +32 -0
  23. data/spec/fixtures/good.ditto +17 -0
  24. data/spec/fixtures/multiversion.ditto +32 -0
  25. data/spec/fixtures/nocomma.ditto +16 -0
  26. data/spec/fixtures/nomethod.ditto +18 -0
  27. data/spec/fixtures/random.ditto +17 -0
  28. data/spec/fixtures/syntax.ditto +19 -0
  29. data/spec/options_spec.rb +43 -25
  30. data/test/data/simple_object.xml +23 -0
  31. data/test/data/simple_object.yaml +39 -0
  32. data/test/ditto/currency.ditto +30 -0
  33. data/test/ditto/currency_group.ditto +17 -0
  34. data/test/dm/currency-1.0.0.dm +13 -0
  35. data/test/dm/currency_group-1.0.0.dm +10 -0
  36. data/test/dm/datamart-1.0 +7 -0
  37. data/test/dm/nested +2 -0
  38. data/test/dmtest.rb +41 -0
  39. data/test/sample.test +2 -0
  40. metadata +59 -42
  41. data/lib/ditto/map.rb +0 -39
data/bin/ditto CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/ruby
1
+ #!/usr/bin/env ruby
2
2
  #
3
3
  require 'ditto/runner'
4
4
  begin
@@ -1,3 +1,4 @@
1
+ @pending
1
2
  Feature: Ditto parses simple entities
2
3
  In order to create test data definitions
3
4
  As a user of Ditto
@@ -1,3 +1,4 @@
1
+ @pending
1
2
  Feature: Ditto loads simple entities
2
3
  In order to create test data
3
4
  As a user of Ditto
@@ -1,3 +1,4 @@
1
+ @pending
1
2
  Feature: Ditto stores entities
2
3
  In order to create test data
3
4
  As a user of Ditto
@@ -65,14 +66,14 @@ Scenario: Ditto stores a single correct entity
65
66
  is_commodity: false
66
67
  """
67
68
  When I run ditto on two files "simple_entity.ditto", "simple_entity.yaml"
68
- Then the exit code should be 0
69
- And the stdout should be
69
+ Then the stdout should be
70
70
  """
71
71
  checked 1 entities, 0 relationships, 0 errors
72
72
  validated 1 entities, 1 instances
73
73
  stored 1 entities, 1 instances
74
74
 
75
75
  """
76
+ And the exit code should be 0
76
77
 
77
78
  Scenario: Ditto stores related entities
78
79
  Given a file named "simple_relationship.ditto" with:
@@ -128,11 +129,11 @@ Scenario: Ditto stores related entities
128
129
  is_commodity: false
129
130
  """
130
131
  When I run ditto on two files "simple_relationship.ditto", "simple_relationship.yaml"
131
- Then the exit code should be 0
132
- And the stdout should be
132
+ Then the stdout should be
133
133
  """
134
134
  checked 2 entities, 1 relationships, 0 errors
135
135
  validated 2 entities, 2 instances
136
136
  stored 2 entities, 2 instances
137
137
 
138
138
  """
139
+ And the exit code should be 0
@@ -1,6 +1,7 @@
1
1
  require 'ditto/version'
2
2
  require 'ditto/options'
3
3
  require 'ditto/entity'
4
+ require 'ditto/error'
4
5
  require 'ditto/dsl'
5
6
  include Ditto::DSL
6
7
  module Ditto
@@ -1,18 +1,23 @@
1
1
  # Define the DSL for Ditto
2
2
  #
3
- require_relative 'entity'
4
- require_relative 'map'
5
-
6
3
  module Ditto
7
4
  module DSL
8
- def version (*args, &block)
9
- Ditto::Entity.set_version(*args, &block)
5
+ def entity (name, version, opts, *methods)
6
+ Ditto::Entity.new(name,version,opts,methods)
10
7
  end
11
- def entity (*args, &block)
12
- Ditto::Entity.define_entity(*args, &block)
8
+ def add (version, &block)
9
+ unless block
10
+ src = Thread.current.backtrace[1].split(':')[0..1]
11
+ raise Error.new(src), "add method missing block (use {} not do/end)"
12
+ end
13
+ [:add, version, block]
13
14
  end
14
- def add (name, opts = {}, &block)
15
- Ditto::Map.add name, opts, &block
15
+ def delete (version, &block)
16
+ unless block
17
+ src = Thread.current.backtrace[1].split(':')[0..1]
18
+ raise Error.new(src), "delete method missing block (use {} not do/end)"
19
+ end
20
+ [:delete, version, block]
16
21
  end
17
22
  end
18
23
  end
@@ -1,143 +1,155 @@
1
1
  # Entity support for Ditto
2
- # Note that Entities are not objects in the current implementation, so we don't
3
- # use a Class, we use a Module.
4
2
  #
3
+ require 'ditto'
4
+ require 'ostruct'
5
+
5
6
  module Ditto
6
- module Entity
7
+ class Entity
7
8
  PROPS = [ :mandatory, :unique, :related ].freeze
8
- @entities = {} # Hash containing definition hashes for each entity
9
- @definitions = {} # Hash remembering where entities were defined
10
- @instances = Hash.new {|h,k| h[k] = []} # Arrays of instance hashes for each entity
11
- @deps = {} # Arrays of dependencies for each entity
12
- @errors = 0
13
-
14
- def self.set_version version
15
- @version = version
9
+ # Entities are stored in a hash keyed by entity name
10
+ # The contents are an array of Entity objects one for each version
11
+ @@entities = Hash.new {|h,k| h[k] = []}
12
+ # Instances are stored in a hash keyed by entity name
13
+ # The contents are a hash (keyed by version) of arrays of instance hashes
14
+ @@instances = Hash.new {|h,k| h[k] = Hash.new{|i,j|i[j] = []}}
15
+
16
+ def self.reset
17
+ @@entities.clear
18
+ @@instances.clear
16
19
  end
17
20
 
21
+ attr_reader :name, :version, :fields, :deps, :loc, :methods
22
+
18
23
  # Maps to the 'entity' DSL keyword
24
+ # Stack depth is just empirically determined! TODO WRITE A TEST!
19
25
  #
20
- def self.define_entity name, details
21
- src = Thread.current.backtrace[2].split(':')[0..1]
22
- src[0] = File.basename(src[0]) unless Ditto::Options.instance.debug
23
- error("entity details must be a hash!", name, src) unless details.kind_of? Hash
24
-
25
- if @entities.has_key?(name.to_sym)
26
- error "duplicate definition of entity", name, src,
27
- "(was previously defined at #{@definitions[name.to_sym].join(':')})"
26
+ def initialize name, version, fields, methods
27
+ src = Thread.current.backtrace[3].split(':')[0..1]
28
+
29
+ @name = name.to_sym
30
+ @version = Gem::Version.new(version) rescue raise(Error.new(src), "#{name}: #{$!.message}")
31
+ @methods = methods
32
+ @loc = src
33
+ @deps = []
34
+
35
+ if @@entities.has_key?(@name) and @@entities[@name].any?{|e| e.version == @version}
36
+ loc = @@entities[@name].select{|e| e.version == @version}.first.loc
37
+ raise Error.new(src), "#{name}: previously defined at #{loc.join(':')}"
28
38
  end
29
- @entities[name.to_sym] = Ditto.symbolize_keys details
30
- @definitions[name.to_sym] = src
31
- end
39
+ raise Error.new(src), "#{name}: entity fields must be a hash!" unless fields.kind_of? Hash
40
+ raise Error.new(src), "#{name}: entity missing add method!" unless methods.size > 0
41
+
42
+ @fields = Ditto.symbolize_keys fields
32
43
 
33
- # Check that the entities are well defined
34
- # return true/false
35
- #
36
- def self.check_definitions verbose = 0
37
- return true if @entities.size == 0
38
- nrel = 0
39
44
  nkey = 0
40
- puts "checking entities..." if verbose > 0
41
- @entities.each do |name,fields|
42
- puts name if verbose > 0
43
- @deps[name] = []
44
- fields.each do |field, props|
45
- symprops = Array(props).select{|p| p.is_a? Symbol}
46
- duffprops = symprops - PROPS
47
- unless duffprops.empty?
48
- error("unknown properties: '#{duffprops.join(' ,')}'", name, @definitions[name])
49
- end
50
- nkey += symprops.count(:unique)
51
- if symprops.include? :related
52
- nrel += 1
53
- if @entities.has_key? field
54
- @deps[name] << field
55
- else
56
- error("unknown relationship to #{field}", name, @definitions[name])
57
- end
58
- end
45
+ @fields.each do |field, props|
46
+ symprops = Array(props).select{|p| p.is_a? Symbol}
47
+ duffprops = symprops - PROPS
48
+ unless duffprops.empty?
49
+ raise Error.new(src), "#{name}: unknown properties: '#{duffprops.join(' ,')}'"
59
50
  end
51
+ nkey += symprops.count(:unique)
52
+ @deps << field if symprops.include? :related
60
53
  end
61
- puts "checked #{@entities.size} entities, #{nrel} relationships, #{@errors} errors"
62
- return error?
63
- end
64
54
 
65
- def self.load_instance name, instance
66
- @instances[name] << instance
55
+ @@entities[@name] << self
67
56
  end
68
57
 
69
- def self.validate_instances verbose = 0
70
- return true if instance_count == 0
71
- puts "checking instances..." if verbose > 0
72
- ninst = nerr = 0
73
- @instances.each do |entity_name,entities|
74
- puts entity_name if verbose > 1
75
- entities.each do |e|
76
- puts e.inspect if verbose > 1
77
- if Ditto::Entity.validate(entity_name, e)
78
- ninst += 1
79
- else
80
- nerr += 1
81
- end
82
- end
58
+ # Errors discovered in here will be load/syntax errors
59
+ # Stack depth is just empirically determined! TODO WRITE A TEST!
60
+ #
61
+ def self.load_from_file f
62
+ begin
63
+ load File.absolute_path(f)
64
+ return true
65
+ rescue Ditto::Error
66
+ raise
67
+ rescue SyntaxError => se # Includes the failing location
68
+ raise Error.new(), "#{se.class}: #{se.to_s}"
69
+ rescue StandardError => le
70
+ loc = le.backtrace[0].split(':')
71
+ raise Error.new(loc), "#{f}: #{le.class}: #{le.to_s}"
83
72
  end
84
- puts "validated #{@instances.keys.size} entities, #{ninst} instances"
85
- return (nerr == 0)
86
73
  end
87
74
 
88
- # Compute dependencies recursively
75
+ # Load data from YAML files into the in-memory representation
76
+ # return number of instances loaded if all OK
89
77
  #
90
- def self.dep_list(name, list, dephash)
91
- if deps = dephash[name]
92
- dephash.delete name
93
- deps.each do |dep|
94
- self.dep_list dep, list, dephash
78
+ def self.load_instances f, verbose = 0
79
+ header = nil
80
+ nent = 0
81
+ YAML::load_documents(File.open(f)) do |doc|
82
+ unless header
83
+ header = Ditto.symbolize_keys doc
84
+ puts "header #{header.to_s}" if verbose > 2
85
+ version = Gem::Version.new(header[:version])
86
+ puts "version: #{version.to_s}" if verbose > 1
87
+ next
95
88
  end
96
- list << name unless list.include? name
97
- else
98
- return if list.include? name # we did it earlier
99
- raise "Missing or circular dependency on #{name}"
89
+ e = doc.flatten
90
+ raise Ditto::Error "Entity instance has multiple keys" if e.size != 2
91
+ name = e[0].to_sym
92
+ fields = Ditto.symbolize_keys e[1]
93
+ puts "#{name}: #{fields.inspect}" if verbose > 1
94
+ @@instances[name][version] << fields
95
+ nent += 1
100
96
  end
97
+ return nent
101
98
  end
102
99
 
103
- # Before storing, sort the instances by dependency
100
+ # Load entities required by the instances
104
101
  #
105
- def self.store_all
106
- seq = []
107
- deps = @deps.dup # shallow copy as deleting entries...
108
- @deps.each_key do |entity|
109
- self.dep_list entity, seq, deps
102
+ def self.load_entities filepath, verbose = 0
103
+ puts "loading entities..." if verbose > 0
104
+ @@instances.each_key do |name|
105
+ self.load_from_file File.expand_path("#{name}.ditto", filepath)
106
+ puts "#{name}: #{@@entities[name].inspect}" if verbose > 1
110
107
  end
111
- return false if Ditto::Map.check_maps(seq, @definitions) > 0
112
- n = Ditto::Map.add_all seq, @instances
113
- puts "stored #{@instances.keys.size} entities, #{n} instances"
114
- return true
115
- end
116
-
117
- def self.validate name, instance
118
- return true
119
- return false unless @entities.has_key?(name)
108
+ puts "loaded #{@@entities.size} entities" if verbose > 0
120
109
  end
121
110
 
122
- def self.instance_count
123
- return @instances.keys.size
111
+ # Store the instances in dependency order
112
+ #
113
+ def self.store_all verbose = 0
114
+ seq = []
115
+ @@entities.each_key { |name| self.dep_list(name, seq, []) }
116
+ puts "dependency sequence: #{seq.inspect}" if verbose > 2
117
+ seq.each do |name|
118
+ @@instances[name].each do |version, instances|
119
+ entity = self.find_entity name, version
120
+ map = self.find_map entity, :add
121
+ instances.each do |instance|
122
+ puts "store #{name}: #{instance.inspect}" if verbose > 2
123
+ map[2].call OpenStruct.new(instance)
124
+ end
125
+ end
126
+ end
124
127
  end
125
128
 
126
- def self.entity_count
127
- return @entities.keys.size
129
+ # Given the input data version get the right entity
130
+ def self.find_entity name, version
131
+ @@entities[name].sort.first
128
132
  end
129
133
 
130
- private
131
- def self.error?
132
- return @errors == 0
134
+ def self.find_map entity, type
135
+ entity.methods.find do |m|
136
+ m[0] == type and true
137
+ end
133
138
  end
134
139
 
135
- def self.error msg, name, src, extra = nil
136
- print "#{name}: #{msg}"
137
- print " at line #{src[1]} in file #{src[0]}" if !src and Ditto::Options.instance.debug
138
- print "\n";
139
- puts extra if extra
140
- @errors += 1
140
+ # Compute dependencies recursively
141
+ # build a list in dependency sequence
142
+ #
143
+ def self.dep_list(name, list, done)
144
+ raise "circular dependency: #{name}" if done.include? name
145
+ raise "missing dependency: #{name}" unless @@entities.has_key? name
146
+ deps = @@entities[name].flat_map {|e| e.deps}.uniq
147
+ done.push name
148
+ deps.each do |depname|
149
+ self.dep_list depname, list, done
150
+ end
151
+ done.pop
152
+ list << name unless list.include? name
141
153
  end
142
154
  end
143
155
  end
@@ -0,0 +1,19 @@
1
+ module Ditto
2
+ class Error < StandardError
3
+ attr_reader :src
4
+ def initialize(src = nil)
5
+ @src = src
6
+ end
7
+ def source debug=false
8
+ return "n/a" if @src.nil? or @src[0].nil? or @src[1].nil?
9
+ @src[0] = File.basename(@src[0]) unless debug
10
+ "#{@src[0]}:#{@src[1]}"
11
+ end
12
+ def message debug=false
13
+ msg = "#{to_s}"
14
+ msg += " at #{source(debug)}"
15
+ msg += "\n#{backtrace.join("\n")}" if debug
16
+ return msg
17
+ end
18
+ end
19
+ end
@@ -4,37 +4,38 @@ require 'singleton'
4
4
 
5
5
  module Ditto
6
6
  class Options < OpenStruct
7
- include Singleton
8
7
 
9
8
  DEFAULTS = {
10
9
  :connstring => ENV['DATABASE_URL'] || 'nodb://a/b',
11
- :modelpath => '.',
10
+ :entitydir => 'ditto',
11
+ :dittomart => ENV['DITTO_MART'],
12
12
  :debug => false,
13
+ :check => false,
13
14
  :droptables => false,
14
15
  :verbose => 0,
15
- :loadpaths => nil
16
+ :loadfiles => nil
16
17
  }
17
- def initialize
18
+ def initialize argv
18
19
  super(DEFAULTS)
19
- end
20
-
21
- def parse argv
22
20
 
23
21
  Ditto::Options::DEFAULTS.each do |k,v|
24
22
  self.send "#{k}=".to_s, v
25
23
  end
26
24
 
27
25
  OptionParser.new do |opts|
28
- opts.banner = "Usage: ditto [ options ] paths..."
29
- opts.separator "Paths will be searched for .ditto files (definitions) and .yaml files (data)"
26
+ opts.banner = "Usage: ditto [ options ] files..."
27
+ opts.separator "Files specified (in yaml format) will be loaded into database"
30
28
  opts.on("-c STRING", "--connection", String, "Database connection string",
31
29
  "Defaults to environment variable DATABASE_URL",
32
30
  "Current default is #{DEFAULTS[:connstring]}") do |s|
33
31
  self.connstring = s
34
32
  end
35
- opts.on("-m PATH", "--modelpath", String, "Path to load DataMapper models from (default #{DEFAULTS[:modelpath]})") do |m|
36
- raise "Modelpath #{m} does not exist!" unless Dir.exist?(m)
37
- self.modelpath = m
33
+ opts.on("-e DIR", "--entitydir", String, "Directory to load Ditto entities from (default #{DEFAULTS[:entitydir]})") do |m|
34
+ raise "Entity dir #{m} does not exist!" unless Dir.exist?(m)
35
+ self.entitydir = m
36
+ end
37
+ opts.on("-m FILE", "--dittomart", String, "File containing Datamapper models for target Datamart (default #{DEFAULTS[:dittomart]})") do |m|
38
+ self.dittomart = m
38
39
  end
39
40
  opts.on("-d", "--debug", "Debug") do |s|
40
41
  self.debug = true
@@ -42,6 +43,9 @@ module Ditto
42
43
  opts.on("-v", "--verbose", "Verbose (repeat for more)") do
43
44
  self.verbose += 1
44
45
  end
46
+ opts.on("--check", "Check given files, don't do anything!") do
47
+ self.check = true
48
+ end
45
49
  opts.on("--droptables", "Delete existing database tables!") do
46
50
  self.droptables = true
47
51
  end
@@ -56,8 +60,36 @@ module Ditto
56
60
  exit(-1)
57
61
  end
58
62
  end
59
- self.loadpaths = argv
63
+ self.martfiles = atfile self.dittomart
64
+ self.loadfiles = argv.map{ |m| atfile m}.flatten
60
65
  return self
61
66
  end
67
+
68
+ # Take a filename and expand it if it's an @file
69
+ # relative paths are always relative to the including file
70
+ # return an array of absolute filenames
71
+ #
72
+ def atfile fn, dir = '.'
73
+ return [] unless fn
74
+ af = fn.split('@',2)
75
+ fn = af[1] if af.size == 2
76
+ fn = File.expand_path(fn,dir)
77
+ raise Errno::ENOENT, fn unless File.exist? fn
78
+ return Array[fn] if af.size == 1
79
+ dir = File.dirname File.absolute_path(fn)
80
+ fh = File.open(fn)
81
+ files = []
82
+ fh.each_with_index do |line,ix|
83
+ line.strip!
84
+ begin
85
+ files.push *atfile(line,dir) unless line =~ /(^\s*$|^\s*#)/
86
+ rescue
87
+ $!.message << " at '#{fn}:#{ix}'"
88
+ raise
89
+ end
90
+ end
91
+ return files
92
+ end
93
+
62
94
  end
63
95
  end