ditto 0.5.0 → 0.6.0

data/bin/ditto

@@ -1,4 +1,4 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 #
 require 'ditto/runner'
 begin

data/features/create_entities.feature

@@ -1,3 +1,4 @@
+@pending
 Feature: Ditto parses simple entities
 In order to create test data definitions
 As a user of Ditto

data/features/load_entity.feature

@@ -1,3 +1,4 @@
+@pending
 Feature: Ditto loads simple entities
 In order to create test data
 As a user of Ditto

data/features/store_entity.feature

@@ -1,3 +1,4 @@
+@pending
 Feature: Ditto stores entities
 In order to create test data
 As a user of Ditto
@@ -65,14 +66,14 @@ Scenario: Ditto stores a single correct entity
     is_commodity: false
   """
   When I run ditto on two files "simple_entity.ditto", "simple_entity.yaml"
-  Then the exit code should be 0
-  And the stdout should be
+  Then the stdout should be
   """
   checked 1 entities, 0 relationships, 0 errors
   validated 1 entities, 1 instances
   stored 1 entities, 1 instances
 
   """
+  And the exit code should be 0
 
 Scenario: Ditto stores related entities
   Given a file named "simple_relationship.ditto" with:
@@ -128,11 +129,11 @@ Scenario: Ditto stores related entities
       is_commodity: false
   """
   When I run ditto on two files "simple_relationship.ditto", "simple_relationship.yaml"
-  Then the exit code should be 0
-  And the stdout should be
+  Then the stdout should be
   """
   checked 2 entities, 1 relationships, 0 errors
   validated 2 entities, 2 instances
   stored 2 entities, 2 instances
 
   """
+  And the exit code should be 0

data/lib/ditto.rb

@@ -1,6 +1,7 @@
 require 'ditto/version'
 require 'ditto/options'
 require 'ditto/entity'
+require 'ditto/error'
 require 'ditto/dsl'
 include Ditto::DSL
 module Ditto

data/lib/ditto/dsl.rb

@@ -1,18 +1,23 @@
 # Define the DSL for Ditto
 #
-require_relative 'entity'
-require_relative 'map'
-
 module Ditto
   module DSL
-    def version (*args, &block)
-      Ditto::Entity.set_version(*args, &block)
+    def entity (name, version, opts, *methods)
+      Ditto::Entity.new(name,version,opts,methods)
     end
-    def entity (*args, &block)
-      Ditto::Entity.define_entity(*args, &block)
+    def add (version, &block)
+      unless block
+	src = Thread.current.backtrace[1].split(':')[0..1]
+	raise Error.new(src), "add method missing block (use {} not do/end)"
+      end
+      [:add, version, block]
     end
-    def add (name, opts = {}, &block)
-      Ditto::Map.add name, opts, &block
+    def delete (version, &block)
+      unless block
+	src = Thread.current.backtrace[1].split(':')[0..1]
+	raise Error.new(src), "delete method missing block (use {} not do/end)"
+      end
+      [:delete, version, block]
     end
   end
 end

data/lib/ditto/entity.rb

@@ -1,143 +1,155 @@
 # Entity support for Ditto
-# Note that Entities are not objects in the current implementation, so we don't
-# use a Class, we use a Module.
 #
+require 'ditto'
+require 'ostruct'
+
 module Ditto
-  module Entity
+  class Entity
     PROPS = [ :mandatory, :unique, :related ].freeze
-    @entities = {} # Hash containing definition hashes for each entity
-    @definitions = {} # Hash remembering where entities were defined
-    @instances = Hash.new {|h,k| h[k] = []} # Arrays of instance hashes for each entity
-    @deps = {} # Arrays of dependencies for each entity
-    @errors = 0
-
-    def self.set_version version
-      @version = version
+    # Entities are stored in a hash keyed by entity name
+    # The contents are an array of Entity objects one for each version
+    @@entities = Hash.new {|h,k| h[k] = []}
+    # Instances are stored in a hash keyed by entity name
+    # The contents are a hash (keyed by version) of arrays of instance hashes
+    @@instances = Hash.new {|h,k| h[k] = Hash.new{|i,j|i[j] = []}}
+
+    def self.reset
+      @@entities.clear
+      @@instances.clear
     end
 
+    attr_reader :name, :version, :fields, :deps, :loc, :methods
+
     # Maps to the 'entity' DSL keyword
+    # Stack depth is just empirically determined! TODO WRITE A TEST!
     #
-    def self.define_entity name, details
-      src = Thread.current.backtrace[2].split(':')[0..1]
-      src[0] = File.basename(src[0]) unless Ditto::Options.instance.debug
-      error("entity details must be a hash!", name, src) unless details.kind_of? Hash
-
-      if @entities.has_key?(name.to_sym)
-	error "duplicate definition of entity", name, src,
-	"(was previously defined at #{@definitions[name.to_sym].join(':')})"
+    def initialize name, version, fields, methods
+      src = Thread.current.backtrace[3].split(':')[0..1]
+
+      @name = name.to_sym
+      @version = Gem::Version.new(version) rescue raise(Error.new(src), "#{name}: #{$!.message}")
+      @methods = methods
+      @loc = src
+      @deps = []
+
+      if @@entities.has_key?(@name) and @@entities[@name].any?{|e| e.version == @version}
+	loc = @@entities[@name].select{|e| e.version == @version}.first.loc
+	raise Error.new(src), "#{name}: previously defined at #{loc.join(':')}"
       end
-      @entities[name.to_sym] = Ditto.symbolize_keys details
-      @definitions[name.to_sym] = src
-    end
+      raise Error.new(src), "#{name}: entity fields must be a hash!" unless fields.kind_of? Hash
+      raise Error.new(src), "#{name}: entity missing add method!" unless methods.size > 0
+
+      @fields = Ditto.symbolize_keys fields
 
-    # Check that the entities are well defined
-    # return true/false
-    #
-    def self.check_definitions verbose = 0
-      return true if @entities.size == 0
-      nrel = 0
       nkey = 0
-      puts "checking entities..." if verbose > 0
-      @entities.each do |name,fields|
-	puts name if verbose > 0
-	@deps[name] = []
-	fields.each do |field, props|
-	  symprops = Array(props).select{|p| p.is_a? Symbol}
-	  duffprops = symprops - PROPS
-	  unless duffprops.empty?
-	    error("unknown properties: '#{duffprops.join(' ,')}'", name, @definitions[name])
-	  end
-	  nkey += symprops.count(:unique)
-	  if symprops.include? :related
-	    nrel += 1
-	    if @entities.has_key? field
-	      @deps[name] << field
-	    else
-	      error("unknown relationship to #{field}", name, @definitions[name])
-	    end
-	  end
+      @fields.each do |field, props|
+	symprops = Array(props).select{|p| p.is_a? Symbol}
+	duffprops = symprops - PROPS
+	unless duffprops.empty?
+	  raise Error.new(src), "#{name}: unknown properties: '#{duffprops.join(' ,')}'"
 	end
+	nkey += symprops.count(:unique)
+	@deps << field if symprops.include? :related
       end
-      puts "checked #{@entities.size} entities, #{nrel} relationships, #{@errors} errors"
-      return error?
-    end
 
-    def self.load_instance name, instance
-      @instances[name] << instance
+      @@entities[@name] << self
     end
 
-    def self.validate_instances verbose = 0
-      return true if instance_count == 0
-      puts "checking instances..." if verbose > 0
-      ninst = nerr = 0
-      @instances.each do |entity_name,entities|
-	puts entity_name if verbose > 1
-	entities.each do |e|
-	  puts e.inspect if verbose > 1
-	  if Ditto::Entity.validate(entity_name, e)
-	    ninst += 1
-	  else
-	    nerr += 1
-	  end
-	end
+    # Errors discovered in here will be load/syntax errors
+    # Stack depth is just empirically determined! TODO WRITE A TEST!
+    #
+    def self.load_from_file f
+      begin
+	load File.absolute_path(f)
+	return true
+      rescue Ditto::Error
+	raise
+      rescue SyntaxError => se # Includes the failing location
+	raise Error.new(), "#{se.class}: #{se.to_s}"
+      rescue StandardError => le
+	loc = le.backtrace[0].split(':')
+	raise Error.new(loc), "#{f}: #{le.class}: #{le.to_s}"
       end
-      puts "validated #{@instances.keys.size} entities, #{ninst} instances"
-      return (nerr == 0)
     end
 
-    # Compute dependencies recursively
+    # Load data from YAML files into the in-memory representation
+    # return number of instances loaded if all OK
     #
-    def self.dep_list(name, list, dephash)
-      if deps = dephash[name]
-	dephash.delete name
-	deps.each do |dep|
-	  self.dep_list dep, list, dephash
+    def self.load_instances f, verbose = 0
+      header = nil
+      nent = 0
+      YAML::load_documents(File.open(f)) do |doc|
+	unless header
+	  header = Ditto.symbolize_keys doc
+	  puts "header #{header.to_s}" if verbose > 2
+	  version = Gem::Version.new(header[:version])
+	  puts "version: #{version.to_s}" if verbose > 1
+	  next
 	end
-	list << name unless list.include? name
-      else
-	return if list.include? name # we did it earlier
-	raise "Missing or circular dependency on #{name}"
+	e = doc.flatten
+	raise Ditto::Error "Entity instance has multiple keys" if e.size != 2
+	name = e[0].to_sym
+	fields = Ditto.symbolize_keys e[1]
+	puts "#{name}: #{fields.inspect}" if verbose > 1
+	@@instances[name][version] << fields
+	nent += 1
       end
+      return nent
     end
 
-    # Before storing, sort the instances by dependency
+    # Load entities required by the instances
     #
-    def self.store_all
-      seq = []
-      deps = @deps.dup # shallow copy as deleting entries...
-      @deps.each_key do |entity|
-	self.dep_list entity, seq, deps
+    def self.load_entities filepath, verbose = 0
+      puts "loading entities..." if verbose > 0
+      @@instances.each_key do |name|
+	self.load_from_file File.expand_path("#{name}.ditto", filepath)
+	puts "#{name}: #{@@entities[name].inspect}" if verbose > 1
       end
-      return false if Ditto::Map.check_maps(seq, @definitions) > 0
-      n = Ditto::Map.add_all seq, @instances
-      puts "stored #{@instances.keys.size} entities, #{n} instances"
-      return true
-    end
-
-    def self.validate name, instance
-      return true
-      return false unless @entities.has_key?(name)
+      puts "loaded #{@@entities.size} entities" if verbose > 0
     end
 
-    def self.instance_count
-      return @instances.keys.size
+    # Store the instances in dependency order
+    #
+    def self.store_all verbose = 0
+      seq = []
+      @@entities.each_key { |name| self.dep_list(name, seq, []) }
+      puts "dependency sequence: #{seq.inspect}" if verbose > 2
+      seq.each do |name|
+	@@instances[name].each do |version, instances|
+	  entity = self.find_entity name, version
+	  map = self.find_map entity, :add
+	  instances.each do |instance|
+	    puts "store #{name}: #{instance.inspect}" if verbose > 2
+	    map[2].call OpenStruct.new(instance)
+	  end
+	end
+      end
     end
 
-    def self.entity_count
-      return @entities.keys.size
+    # Given the input data version get the right entity
+    def self.find_entity name, version
+       @@entities[name].sort.first
     end
 
-  private
-    def self.error?
-      return @errors == 0
+    def self.find_map entity, type
+      entity.methods.find do |m|
+	m[0] == type and true
+      end
     end
 
-    def self.error msg, name, src, extra = nil
-      print "#{name}: #{msg}"
-      print " at line #{src[1]} in file #{src[0]}" if !src and Ditto::Options.instance.debug
-      print "\n";
-      puts extra if extra
-      @errors += 1
+    # Compute dependencies recursively
+    # build a list in dependency sequence
+    #
+    def self.dep_list(name, list, done)
+      raise "circular dependency: #{name}" if done.include? name
+      raise "missing dependency: #{name}" unless @@entities.has_key? name
+      deps = @@entities[name].flat_map {|e| e.deps}.uniq
+      done.push name
+      deps.each do |depname|
+	self.dep_list depname, list, done
+      end
+      done.pop
+      list << name unless list.include? name
     end
   end
 end

data/lib/ditto/error.rb

@@ -0,0 +1,19 @@
+module Ditto
+  class Error < StandardError
+    attr_reader :src
+    def initialize(src = nil)
+      @src = src
+    end
+    def source debug=false
+      return "n/a" if @src.nil? or @src[0].nil? or @src[1].nil?
+      @src[0] = File.basename(@src[0]) unless debug
+      "#{@src[0]}:#{@src[1]}"
+    end
+    def message debug=false
+      msg = "#{to_s}"
+      msg += " at #{source(debug)}"
+      msg += "\n#{backtrace.join("\n")}" if debug
+      return msg
+    end
+  end
+end

data/lib/ditto/options.rb

@@ -4,37 +4,38 @@ require 'singleton'
 
 module Ditto
   class Options < OpenStruct
-    include Singleton
 
     DEFAULTS = {
 	:connstring => ENV['DATABASE_URL'] || 'nodb://a/b',
-	:modelpath  => '.',
+	:entitydir  => 'ditto',
+	:dittomart  => ENV['DITTO_MART'],
 	:debug	    => false,
+	:check	    => false,
 	:droptables => false,
 	:verbose    => 0,
-	:loadpaths  => nil
+	:loadfiles  => nil
     }
-    def initialize
+    def initialize argv
       super(DEFAULTS)
-    end
-
-    def parse argv
 
       Ditto::Options::DEFAULTS.each do |k,v|
 	self.send "#{k}=".to_s, v
       end
 
       OptionParser.new do |opts|
-	opts.banner = "Usage: ditto [ options ] paths..."
-	opts.separator "Paths will be searched for .ditto files (definitions) and .yaml files (data)"
+	opts.banner = "Usage: ditto [ options ] files..."
+	opts.separator "Files specified (in yaml format) will be loaded into database"
 	opts.on("-c STRING", "--connection", String, "Database connection string",
 		"Defaults to environment variable DATABASE_URL",
 		"Current default is #{DEFAULTS[:connstring]}") do |s|
 	  self.connstring = s
 	end
-	opts.on("-m PATH", "--modelpath", String, "Path to load DataMapper models from (default #{DEFAULTS[:modelpath]})") do |m|
-	  raise "Modelpath #{m} does not exist!" unless Dir.exist?(m)
-	  self.modelpath = m
+	opts.on("-e DIR", "--entitydir", String, "Directory to load Ditto entities from (default #{DEFAULTS[:entitydir]})") do |m|
+	  raise "Entity dir #{m} does not exist!" unless Dir.exist?(m)
+	  self.entitydir = m
+	end
+	opts.on("-m FILE", "--dittomart", String, "File containing Datamapper models for target Datamart (default #{DEFAULTS[:dittomart]})") do |m|
+	  self.dittomart = m
 	end
 	opts.on("-d", "--debug", "Debug") do |s|
 	  self.debug = true
@@ -42,6 +43,9 @@ module Ditto
 	opts.on("-v", "--verbose", "Verbose (repeat for more)") do
 	  self.verbose += 1
 	end
+	opts.on("--check", "Check given files, don't do anything!") do
+	  self.check = true
+	end
 	opts.on("--droptables", "Delete existing database tables!") do
 	  self.droptables = true
 	end
@@ -56,8 +60,36 @@ module Ditto
 	  exit(-1)
 	end
       end
-      self.loadpaths = argv
+      self.martfiles = atfile self.dittomart
+      self.loadfiles = argv.map{ |m| atfile m}.flatten
       return self
     end
+
+    # Take a filename and expand it if it's an @file
+    # relative paths are always relative to the including file
+    # return an array of absolute filenames
+    #
+    def atfile fn, dir = '.'
+      return [] unless fn
+      af = fn.split('@',2)
+      fn = af[1] if af.size == 2
+      fn = File.expand_path(fn,dir)
+      raise Errno::ENOENT, fn unless File.exist? fn
+      return Array[fn] if af.size == 1
+      dir = File.dirname File.absolute_path(fn)
+      fh = File.open(fn)
+      files = []
+      fh.each_with_index do |line,ix|
+	line.strip!
+	begin
+	  files.push *atfile(line,dir) unless line =~ /(^\s*$|^\s*#)/
+	rescue
+	  $!.message << " at '#{fn}:#{ix}'"
+	  raise
+	end
+      end
+      return files
+    end
+
   end
 end