ditto 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/ditto +1 -1
- data/features/create_entities.feature +1 -0
- data/features/load_entity.feature +1 -0
- data/features/store_entity.feature +5 -4
- data/lib/ditto.rb +1 -0
- data/lib/ditto/dsl.rb +14 -9
- data/lib/ditto/entity.rb +118 -106
- data/lib/ditto/error.rb +19 -0
- data/lib/ditto/options.rb +45 -13
- data/lib/ditto/runner.rb +37 -56
- data/lib/ditto/version.rb +1 -1
- data/spec/entity_spec.rb +83 -59
- data/spec/fixtures/badmap.ditto +18 -0
- data/spec/fixtures/badver.ditto +17 -0
- data/spec/fixtures/bigcircle.ditto +17 -0
- data/spec/fixtures/circle.ditto +16 -0
- data/spec/fixtures/continent.ditto +16 -0
- data/spec/fixtures/country.ditto +16 -0
- data/spec/fixtures/currency.ditto +30 -0
- data/spec/fixtures/currency_group.ditto +17 -0
- data/spec/fixtures/dot.ditto +17 -0
- data/spec/fixtures/dupversion.ditto +32 -0
- data/spec/fixtures/good.ditto +17 -0
- data/spec/fixtures/multiversion.ditto +32 -0
- data/spec/fixtures/nocomma.ditto +16 -0
- data/spec/fixtures/nomethod.ditto +18 -0
- data/spec/fixtures/random.ditto +17 -0
- data/spec/fixtures/syntax.ditto +19 -0
- data/spec/options_spec.rb +43 -25
- data/test/data/simple_object.xml +23 -0
- data/test/data/simple_object.yaml +39 -0
- data/test/ditto/currency.ditto +30 -0
- data/test/ditto/currency_group.ditto +17 -0
- data/test/dm/currency-1.0.0.dm +13 -0
- data/test/dm/currency_group-1.0.0.dm +10 -0
- data/test/dm/datamart-1.0 +7 -0
- data/test/dm/nested +2 -0
- data/test/dmtest.rb +41 -0
- data/test/sample.test +2 -0
- metadata +59 -42
- data/lib/ditto/map.rb +0 -39
data/bin/ditto
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
@pending
|
1
2
|
Feature: Ditto stores entities
|
2
3
|
In order to create test data
|
3
4
|
As a user of Ditto
|
@@ -65,14 +66,14 @@ Scenario: Ditto stores a single correct entity
|
|
65
66
|
is_commodity: false
|
66
67
|
"""
|
67
68
|
When I run ditto on two files "simple_entity.ditto", "simple_entity.yaml"
|
68
|
-
Then the
|
69
|
-
And the stdout should be
|
69
|
+
Then the stdout should be
|
70
70
|
"""
|
71
71
|
checked 1 entities, 0 relationships, 0 errors
|
72
72
|
validated 1 entities, 1 instances
|
73
73
|
stored 1 entities, 1 instances
|
74
74
|
|
75
75
|
"""
|
76
|
+
And the exit code should be 0
|
76
77
|
|
77
78
|
Scenario: Ditto stores related entities
|
78
79
|
Given a file named "simple_relationship.ditto" with:
|
@@ -128,11 +129,11 @@ Scenario: Ditto stores related entities
|
|
128
129
|
is_commodity: false
|
129
130
|
"""
|
130
131
|
When I run ditto on two files "simple_relationship.ditto", "simple_relationship.yaml"
|
131
|
-
Then the
|
132
|
-
And the stdout should be
|
132
|
+
Then the stdout should be
|
133
133
|
"""
|
134
134
|
checked 2 entities, 1 relationships, 0 errors
|
135
135
|
validated 2 entities, 2 instances
|
136
136
|
stored 2 entities, 2 instances
|
137
137
|
|
138
138
|
"""
|
139
|
+
And the exit code should be 0
|
data/lib/ditto.rb
CHANGED
data/lib/ditto/dsl.rb
CHANGED
@@ -1,18 +1,23 @@
|
|
1
1
|
# Define the DSL for Ditto
|
2
2
|
#
|
3
|
-
require_relative 'entity'
|
4
|
-
require_relative 'map'
|
5
|
-
|
6
3
|
module Ditto
|
7
4
|
module DSL
|
8
|
-
def
|
9
|
-
Ditto::Entity.
|
5
|
+
def entity (name, version, opts, *methods)
|
6
|
+
Ditto::Entity.new(name,version,opts,methods)
|
10
7
|
end
|
11
|
-
def
|
12
|
-
|
8
|
+
def add (version, &block)
|
9
|
+
unless block
|
10
|
+
src = Thread.current.backtrace[1].split(':')[0..1]
|
11
|
+
raise Error.new(src), "add method missing block (use {} not do/end)"
|
12
|
+
end
|
13
|
+
[:add, version, block]
|
13
14
|
end
|
14
|
-
def
|
15
|
-
|
15
|
+
def delete (version, &block)
|
16
|
+
unless block
|
17
|
+
src = Thread.current.backtrace[1].split(':')[0..1]
|
18
|
+
raise Error.new(src), "delete method missing block (use {} not do/end)"
|
19
|
+
end
|
20
|
+
[:delete, version, block]
|
16
21
|
end
|
17
22
|
end
|
18
23
|
end
|
data/lib/ditto/entity.rb
CHANGED
@@ -1,143 +1,155 @@
|
|
1
1
|
# Entity support for Ditto
|
2
|
-
# Note that Entities are not objects in the current implementation, so we don't
|
3
|
-
# use a Class, we use a Module.
|
4
2
|
#
|
3
|
+
require 'ditto'
|
4
|
+
require 'ostruct'
|
5
|
+
|
5
6
|
module Ditto
|
6
|
-
|
7
|
+
class Entity
|
7
8
|
PROPS = [ :mandatory, :unique, :related ].freeze
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
9
|
+
# Entities are stored in a hash keyed by entity name
|
10
|
+
# The contents are an array of Entity objects one for each version
|
11
|
+
@@entities = Hash.new {|h,k| h[k] = []}
|
12
|
+
# Instances are stored in a hash keyed by entity name
|
13
|
+
# The contents are a hash (keyed by version) of arrays of instance hashes
|
14
|
+
@@instances = Hash.new {|h,k| h[k] = Hash.new{|i,j|i[j] = []}}
|
15
|
+
|
16
|
+
def self.reset
|
17
|
+
@@entities.clear
|
18
|
+
@@instances.clear
|
16
19
|
end
|
17
20
|
|
21
|
+
attr_reader :name, :version, :fields, :deps, :loc, :methods
|
22
|
+
|
18
23
|
# Maps to the 'entity' DSL keyword
|
24
|
+
# Stack depth is just empirically determined! TODO WRITE A TEST!
|
19
25
|
#
|
20
|
-
def
|
21
|
-
src = Thread.current.backtrace[
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
26
|
+
def initialize name, version, fields, methods
|
27
|
+
src = Thread.current.backtrace[3].split(':')[0..1]
|
28
|
+
|
29
|
+
@name = name.to_sym
|
30
|
+
@version = Gem::Version.new(version) rescue raise(Error.new(src), "#{name}: #{$!.message}")
|
31
|
+
@methods = methods
|
32
|
+
@loc = src
|
33
|
+
@deps = []
|
34
|
+
|
35
|
+
if @@entities.has_key?(@name) and @@entities[@name].any?{|e| e.version == @version}
|
36
|
+
loc = @@entities[@name].select{|e| e.version == @version}.first.loc
|
37
|
+
raise Error.new(src), "#{name}: previously defined at #{loc.join(':')}"
|
28
38
|
end
|
29
|
-
|
30
|
-
|
31
|
-
|
39
|
+
raise Error.new(src), "#{name}: entity fields must be a hash!" unless fields.kind_of? Hash
|
40
|
+
raise Error.new(src), "#{name}: entity missing add method!" unless methods.size > 0
|
41
|
+
|
42
|
+
@fields = Ditto.symbolize_keys fields
|
32
43
|
|
33
|
-
# Check that the entities are well defined
|
34
|
-
# return true/false
|
35
|
-
#
|
36
|
-
def self.check_definitions verbose = 0
|
37
|
-
return true if @entities.size == 0
|
38
|
-
nrel = 0
|
39
44
|
nkey = 0
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
symprops = Array(props).select{|p| p.is_a? Symbol}
|
46
|
-
duffprops = symprops - PROPS
|
47
|
-
unless duffprops.empty?
|
48
|
-
error("unknown properties: '#{duffprops.join(' ,')}'", name, @definitions[name])
|
49
|
-
end
|
50
|
-
nkey += symprops.count(:unique)
|
51
|
-
if symprops.include? :related
|
52
|
-
nrel += 1
|
53
|
-
if @entities.has_key? field
|
54
|
-
@deps[name] << field
|
55
|
-
else
|
56
|
-
error("unknown relationship to #{field}", name, @definitions[name])
|
57
|
-
end
|
58
|
-
end
|
45
|
+
@fields.each do |field, props|
|
46
|
+
symprops = Array(props).select{|p| p.is_a? Symbol}
|
47
|
+
duffprops = symprops - PROPS
|
48
|
+
unless duffprops.empty?
|
49
|
+
raise Error.new(src), "#{name}: unknown properties: '#{duffprops.join(' ,')}'"
|
59
50
|
end
|
51
|
+
nkey += symprops.count(:unique)
|
52
|
+
@deps << field if symprops.include? :related
|
60
53
|
end
|
61
|
-
puts "checked #{@entities.size} entities, #{nrel} relationships, #{@errors} errors"
|
62
|
-
return error?
|
63
|
-
end
|
64
54
|
|
65
|
-
|
66
|
-
@instances[name] << instance
|
55
|
+
@@entities[@name] << self
|
67
56
|
end
|
68
57
|
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
58
|
+
# Errors discovered in here will be load/syntax errors
|
59
|
+
# Stack depth is just empirically determined! TODO WRITE A TEST!
|
60
|
+
#
|
61
|
+
def self.load_from_file f
|
62
|
+
begin
|
63
|
+
load File.absolute_path(f)
|
64
|
+
return true
|
65
|
+
rescue Ditto::Error
|
66
|
+
raise
|
67
|
+
rescue SyntaxError => se # Includes the failing location
|
68
|
+
raise Error.new(), "#{se.class}: #{se.to_s}"
|
69
|
+
rescue StandardError => le
|
70
|
+
loc = le.backtrace[0].split(':')
|
71
|
+
raise Error.new(loc), "#{f}: #{le.class}: #{le.to_s}"
|
83
72
|
end
|
84
|
-
puts "validated #{@instances.keys.size} entities, #{ninst} instances"
|
85
|
-
return (nerr == 0)
|
86
73
|
end
|
87
74
|
|
88
|
-
#
|
75
|
+
# Load data from YAML files into the in-memory representation
|
76
|
+
# return number of instances loaded if all OK
|
89
77
|
#
|
90
|
-
def self.
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
78
|
+
def self.load_instances f, verbose = 0
|
79
|
+
header = nil
|
80
|
+
nent = 0
|
81
|
+
YAML::load_documents(File.open(f)) do |doc|
|
82
|
+
unless header
|
83
|
+
header = Ditto.symbolize_keys doc
|
84
|
+
puts "header #{header.to_s}" if verbose > 2
|
85
|
+
version = Gem::Version.new(header[:version])
|
86
|
+
puts "version: #{version.to_s}" if verbose > 1
|
87
|
+
next
|
95
88
|
end
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
89
|
+
e = doc.flatten
|
90
|
+
raise Ditto::Error "Entity instance has multiple keys" if e.size != 2
|
91
|
+
name = e[0].to_sym
|
92
|
+
fields = Ditto.symbolize_keys e[1]
|
93
|
+
puts "#{name}: #{fields.inspect}" if verbose > 1
|
94
|
+
@@instances[name][version] << fields
|
95
|
+
nent += 1
|
100
96
|
end
|
97
|
+
return nent
|
101
98
|
end
|
102
99
|
|
103
|
-
#
|
100
|
+
# Load entities required by the instances
|
104
101
|
#
|
105
|
-
def self.
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
102
|
+
def self.load_entities filepath, verbose = 0
|
103
|
+
puts "loading entities..." if verbose > 0
|
104
|
+
@@instances.each_key do |name|
|
105
|
+
self.load_from_file File.expand_path("#{name}.ditto", filepath)
|
106
|
+
puts "#{name}: #{@@entities[name].inspect}" if verbose > 1
|
110
107
|
end
|
111
|
-
|
112
|
-
n = Ditto::Map.add_all seq, @instances
|
113
|
-
puts "stored #{@instances.keys.size} entities, #{n} instances"
|
114
|
-
return true
|
115
|
-
end
|
116
|
-
|
117
|
-
def self.validate name, instance
|
118
|
-
return true
|
119
|
-
return false unless @entities.has_key?(name)
|
108
|
+
puts "loaded #{@@entities.size} entities" if verbose > 0
|
120
109
|
end
|
121
110
|
|
122
|
-
|
123
|
-
|
111
|
+
# Store the instances in dependency order
|
112
|
+
#
|
113
|
+
def self.store_all verbose = 0
|
114
|
+
seq = []
|
115
|
+
@@entities.each_key { |name| self.dep_list(name, seq, []) }
|
116
|
+
puts "dependency sequence: #{seq.inspect}" if verbose > 2
|
117
|
+
seq.each do |name|
|
118
|
+
@@instances[name].each do |version, instances|
|
119
|
+
entity = self.find_entity name, version
|
120
|
+
map = self.find_map entity, :add
|
121
|
+
instances.each do |instance|
|
122
|
+
puts "store #{name}: #{instance.inspect}" if verbose > 2
|
123
|
+
map[2].call OpenStruct.new(instance)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
124
127
|
end
|
125
128
|
|
126
|
-
|
127
|
-
|
129
|
+
# Given the input data version get the right entity
|
130
|
+
def self.find_entity name, version
|
131
|
+
@@entities[name].sort.first
|
128
132
|
end
|
129
133
|
|
130
|
-
|
131
|
-
|
132
|
-
|
134
|
+
def self.find_map entity, type
|
135
|
+
entity.methods.find do |m|
|
136
|
+
m[0] == type and true
|
137
|
+
end
|
133
138
|
end
|
134
139
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
140
|
+
# Compute dependencies recursively
|
141
|
+
# build a list in dependency sequence
|
142
|
+
#
|
143
|
+
def self.dep_list(name, list, done)
|
144
|
+
raise "circular dependency: #{name}" if done.include? name
|
145
|
+
raise "missing dependency: #{name}" unless @@entities.has_key? name
|
146
|
+
deps = @@entities[name].flat_map {|e| e.deps}.uniq
|
147
|
+
done.push name
|
148
|
+
deps.each do |depname|
|
149
|
+
self.dep_list depname, list, done
|
150
|
+
end
|
151
|
+
done.pop
|
152
|
+
list << name unless list.include? name
|
141
153
|
end
|
142
154
|
end
|
143
155
|
end
|
data/lib/ditto/error.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
module Ditto
|
2
|
+
class Error < StandardError
|
3
|
+
attr_reader :src
|
4
|
+
def initialize(src = nil)
|
5
|
+
@src = src
|
6
|
+
end
|
7
|
+
def source debug=false
|
8
|
+
return "n/a" if @src.nil? or @src[0].nil? or @src[1].nil?
|
9
|
+
@src[0] = File.basename(@src[0]) unless debug
|
10
|
+
"#{@src[0]}:#{@src[1]}"
|
11
|
+
end
|
12
|
+
def message debug=false
|
13
|
+
msg = "#{to_s}"
|
14
|
+
msg += " at #{source(debug)}"
|
15
|
+
msg += "\n#{backtrace.join("\n")}" if debug
|
16
|
+
return msg
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
data/lib/ditto/options.rb
CHANGED
@@ -4,37 +4,38 @@ require 'singleton'
|
|
4
4
|
|
5
5
|
module Ditto
|
6
6
|
class Options < OpenStruct
|
7
|
-
include Singleton
|
8
7
|
|
9
8
|
DEFAULTS = {
|
10
9
|
:connstring => ENV['DATABASE_URL'] || 'nodb://a/b',
|
11
|
-
:
|
10
|
+
:entitydir => 'ditto',
|
11
|
+
:dittomart => ENV['DITTO_MART'],
|
12
12
|
:debug => false,
|
13
|
+
:check => false,
|
13
14
|
:droptables => false,
|
14
15
|
:verbose => 0,
|
15
|
-
:
|
16
|
+
:loadfiles => nil
|
16
17
|
}
|
17
|
-
def initialize
|
18
|
+
def initialize argv
|
18
19
|
super(DEFAULTS)
|
19
|
-
end
|
20
|
-
|
21
|
-
def parse argv
|
22
20
|
|
23
21
|
Ditto::Options::DEFAULTS.each do |k,v|
|
24
22
|
self.send "#{k}=".to_s, v
|
25
23
|
end
|
26
24
|
|
27
25
|
OptionParser.new do |opts|
|
28
|
-
opts.banner = "Usage: ditto [ options ]
|
29
|
-
opts.separator "
|
26
|
+
opts.banner = "Usage: ditto [ options ] files..."
|
27
|
+
opts.separator "Files specified (in yaml format) will be loaded into database"
|
30
28
|
opts.on("-c STRING", "--connection", String, "Database connection string",
|
31
29
|
"Defaults to environment variable DATABASE_URL",
|
32
30
|
"Current default is #{DEFAULTS[:connstring]}") do |s|
|
33
31
|
self.connstring = s
|
34
32
|
end
|
35
|
-
opts.on("-
|
36
|
-
raise "
|
37
|
-
self.
|
33
|
+
opts.on("-e DIR", "--entitydir", String, "Directory to load Ditto entities from (default #{DEFAULTS[:entitydir]})") do |m|
|
34
|
+
raise "Entity dir #{m} does not exist!" unless Dir.exist?(m)
|
35
|
+
self.entitydir = m
|
36
|
+
end
|
37
|
+
opts.on("-m FILE", "--dittomart", String, "File containing Datamapper models for target Datamart (default #{DEFAULTS[:dittomart]})") do |m|
|
38
|
+
self.dittomart = m
|
38
39
|
end
|
39
40
|
opts.on("-d", "--debug", "Debug") do |s|
|
40
41
|
self.debug = true
|
@@ -42,6 +43,9 @@ module Ditto
|
|
42
43
|
opts.on("-v", "--verbose", "Verbose (repeat for more)") do
|
43
44
|
self.verbose += 1
|
44
45
|
end
|
46
|
+
opts.on("--check", "Check given files, don't do anything!") do
|
47
|
+
self.check = true
|
48
|
+
end
|
45
49
|
opts.on("--droptables", "Delete existing database tables!") do
|
46
50
|
self.droptables = true
|
47
51
|
end
|
@@ -56,8 +60,36 @@ module Ditto
|
|
56
60
|
exit(-1)
|
57
61
|
end
|
58
62
|
end
|
59
|
-
self.
|
63
|
+
self.martfiles = atfile self.dittomart
|
64
|
+
self.loadfiles = argv.map{ |m| atfile m}.flatten
|
60
65
|
return self
|
61
66
|
end
|
67
|
+
|
68
|
+
# Take a filename and expand it if it's an @file
|
69
|
+
# relative paths are always relative to the including file
|
70
|
+
# return an array of absolute filenames
|
71
|
+
#
|
72
|
+
def atfile fn, dir = '.'
|
73
|
+
return [] unless fn
|
74
|
+
af = fn.split('@',2)
|
75
|
+
fn = af[1] if af.size == 2
|
76
|
+
fn = File.expand_path(fn,dir)
|
77
|
+
raise Errno::ENOENT, fn unless File.exist? fn
|
78
|
+
return Array[fn] if af.size == 1
|
79
|
+
dir = File.dirname File.absolute_path(fn)
|
80
|
+
fh = File.open(fn)
|
81
|
+
files = []
|
82
|
+
fh.each_with_index do |line,ix|
|
83
|
+
line.strip!
|
84
|
+
begin
|
85
|
+
files.push *atfile(line,dir) unless line =~ /(^\s*$|^\s*#)/
|
86
|
+
rescue
|
87
|
+
$!.message << " at '#{fn}:#{ix}'"
|
88
|
+
raise
|
89
|
+
end
|
90
|
+
end
|
91
|
+
return files
|
92
|
+
end
|
93
|
+
|
62
94
|
end
|
63
95
|
end
|