gd_bam 0.0.15 → 0.1.0

data/lib/bam/version.rb

@@ -1,3 +1,3 @@
 module Bam
-  VERSION = '0.0.15'
+  VERSION = '0.1.0'
 end

data/lib/bam.rb

@@ -3,6 +3,57 @@ require 'pry'
 require 'zip/zip'
 require 'fileutils'
 $:.unshift(File.dirname(__FILE__))
+require 'utils/utils'
+require 'compatibility'
+
+require 'cloud_connect/dsl/helpers'
+require 'cloud_connect/dsl/sf_helpers'
+require 'cloud_connect/dsl/es_helpers'
+require 'cloud_connect/dsl/structure_helpers'
+require 'cloud_connect/dsl/cc'
+require 'cloud_connect/dsl/nodes'
+
+require 'base/errors'
+require 'base/tap'
+require 'base/taps'
+require 'base/sink'
+require 'base/flow'
+require 'base/metadata'
+require 'base/project'
+require 'base/graph'
+require 'base/step'
+require 'base/repo'
+
+require 'dsl/dsl'
+require 'compiler/compiler'
+require 'compiler/etl_visitor'
+
+require 'commands/commands'
+require 'commands/scaffold_commands'
+require 'commands/sf_commands'
+require 'commands/deployment'
+require 'commands/gd_commands'
+require 'commands/docs_commands'
+require 'commands/validators'
+
+require 'generators/downloaders'
+require 'generators/validators'
+require 'generators/etl'
+
 require 'runtime'
+
+
 # Add requires for other files you add to your project here, so
 # you just need to require this one file in your bin file
+
+module GoodData
+  module Bam
+    
+    GENERATOR_ROOT = Pathname(__FILE__).expand_path.dirname
+    BAM_ROOT = GENERATOR_ROOT.parent
+
+    BAM_DEFINED_GRAPHS_ROOT = GENERATOR_ROOT + "graphs"
+    REPO_ROOT = GENERATOR_ROOT + "repo"
+    TEMPLATE_ROOT = BAM_ROOT + "templates"
+  end
+end

data/lib/base/errors.rb

@@ -0,0 +1,15 @@
+module GoodData 
+  module Bam
+    class RemoveMetadataFieldError < RuntimeError
+
+      attr_reader :options, :metadata, :field
+
+      def initialize(message, options={})
+        super(message)
+        @options = options
+        @metadata = options[:metadata]
+        @field = options[:field]
+      end
+    end
+  end
+end

data/lib/base/flow.rb

@@ -0,0 +1,37 @@
+module GoodData 
+  module Bam
+    module Flow
+
+      def self.create(flow)
+        {:steps => []}.merge(flow).merge({:type => :flow})
+      end
+
+      attr_accessor :steps, :name
+
+      def self.define(name="", &script)
+        # puts "Reading flow #{name}"
+        x = self.new
+        x.flow_name(name)
+        x.instance_eval(&script)
+        x
+      end
+
+      def self.add_step(flow, step)
+        new_steps = flow[:steps].concat([step])
+        Flow.create(flow.merge({:steps => new_steps}))
+      end
+
+      def self.remove_last_step(flow)
+        new_steps = flow[:steps][0..-2]
+        Flow.create(flow.merge({:steps => new_steps}))
+      end
+
+      # DSL is mutating part of the API. DO not use it except in DSL part of BAM that means in the flows
+
+    end
+
+    def self.flow(name="", &bl)
+      Flow.define(name, &bl)
+    end
+  end
+end

data/lib/base/graph.rb

@@ -0,0 +1,23 @@
+module GoodData 
+  module Bam
+    module Graph
+
+      def self.create(graph)
+        fail "Graph should have graph to execute. You defined \"#{graph}\"" if graph[:path].blank?
+        {
+          :inputs => [],
+          :outputs => []
+        }.merge(graph).merge({
+          :path => Pathname(graph[:path]),
+          :name => Pathname(graph[:path]).basename.to_s,
+          :type => :graph
+        })
+      end
+
+      def self.get_path(graph)
+        Pathname(graph[:repository][:base]) + graph[:path]
+      end
+
+    end
+  end
+end

data/lib/base/metadata.rb

@@ -0,0 +1,107 @@
+module GoodData 
+  module Bam
+    module Metadata
+
+      attr_accessor :attributes
+
+      def self.create(metadata)
+        metadata
+      end
+
+      def self.clone(metadata)
+        Metadata.create(attributes)
+      end
+
+      def self.find_field(metadata, name)
+        metadata[:fields].detect {|f| f[:name] == name}
+      end
+
+      def self.has_field?(metadata, name)
+        !Metadata.find_field(metadata, name).nil?
+      end
+
+      def self.add_field(metadata, field)
+        fail "You have to specify name at the metadata change. You specified \"#{field}\"" unless field.respond_to?(:has_key?) && field.has_key?(:name)
+        position = field[:position] || 0
+        what = {
+          :name => field[:name],
+          :type => field[:type]
+        }
+        fields = metadata[:fields].clone
+        fields.insert(position - 1, what) unless Metadata.has_field?(metadata, field[:name])
+        Metadata.create(metadata.merge({:fields => fields}))
+      end
+
+      def self.remove_field(metadata, field_name)
+        fields = metadata[:fields]
+        fail RemoveMetadataFieldError.new("When trying to remove field \"#{field_name}\" from metadata \"#{metadata[:name]}\" the field was not found", :field => field_name, :metadata => metadata) unless fields.detect {|f| f[:name] == field_name}
+        fields = fields.find_all {|f| f[:name] != field_name}
+        Metadata.create(metadata.merge({:fields => fields}))
+      end
+
+      def self.get_metadata(what)
+        {
+          :input_metadata => Metadata.get_source_metadata(what),
+          :output_metadata => Metadata.get_target_metadata(what)
+        }
+      end
+
+      def self.get_target_metadata(what, options={})
+        name = options[:name] || what[:id]
+        case what[:type]
+        when :sink
+          Metadata.create({
+            :name   => name,
+            :id     => name,
+            :fields => GoodData::CloverGenerator::clover_metadata_for_fields(what[:fields].map {|f| {:name => f[:meta]}})
+          })
+        when :tap, :graph
+          Metadata.create({
+            :name   => name,
+            :id     => name,
+            :fields => GoodData::CloverGenerator::clover_metadata_for_module(what)
+          })
+        end
+      end
+      
+      def self.get_source_metadata(what, options={})
+        name = options[:name] || what[:id]
+        case what[:type]
+        when :sink
+          Metadata.create({
+            :name   => name,
+            :id     => name,
+            :fields => GoodData::CloverGenerator::clover_metadata_for_fields(what[:fields].map {|f| {:name => f[:meta]}})
+          })
+        when :tap, :graph
+          Metadata.create({
+            :name   => name,
+            :id     => name,
+            :fields => GoodData::CloverGenerator::sf_metadata_for_module(what)
+          })
+        end
+      end
+
+      def self.add_timestamp_as_date(metadata)
+        if GoodData::Bam::Metadata.has_field?(metadata, "Timestamp")
+          metadata = GoodData::Bam::Metadata.remove_field(metadata, "Timestamp")
+        end
+        GoodData::Bam::Metadata.add_field(metadata, {:name => "Timestamp", :type => "date"})
+      end
+
+      def self.remove_timestamp(metadata)
+        if GoodData::Bam::Metadata.has_field?(metadata, "Timestamp")
+          GoodData::Bam::Metadata.remove_field(metadata, "Timestamp")
+        else
+          metadata
+        end
+      end
+
+      # def change
+      #   yield(self)
+      #   self
+      # end
+
+    end
+  end
+end

data/lib/base/project.rb

@@ -0,0 +1,95 @@
+module GoodData 
+  module Bam
+    module Project
+
+      def self.create(project)
+        {
+          :flows => [],
+          :flows_definitions => [],
+          :taps => [],
+          :sinks => []
+        }.merge(project).merge({:type => :project})
+      end
+
+      def self.find_by(project, what, by_what, name)
+        project[what].detect {|item| item[by_what] == name}
+      end
+
+      def self.find_tap_by_id(project, name)
+        Project.find_by(project, :taps, :id, name)
+      end
+
+      def self.find_tap_by_name(project, name)
+        Project.find_by(project, :taps, :name, name)
+      end
+
+      def self.find_sink_by_id(project, name)
+        Project.find_by(project, :sinks, :id, name)
+      end
+
+      def self.find_sink_by_name(project, name)
+        Project.find_by(project, :sinks, :name, name)
+      end
+
+      def self.find_graph_by_path(project, path)
+        found = project[:graph_repos].reduce([]) do |acc, repo|
+          if Bam::Repository.find(repo, path)
+            acc.concat([Bam::Repository.find(repo, path)])
+          else
+            acc
+          end
+        end
+        found.first
+      end
+
+      def self.load_taps(project, path="./taps/")
+        p = Pathname(path)
+        taps = Pathname.glob(p + "*.json").map do |p|
+          Tap.create(JSON.parse(File.read(p), :symbolize_names => true))
+        end
+        Project.create(project.merge({:taps => taps.map {|tap| Tap.create(tap)}}))
+      end
+
+      def self.set_graph_repos(project, repos=[])
+        
+        # paths = paths.map {|p| Pathname(p)}
+        #         graphs = paths.reduce([]) do |acc, path|
+        #           Pathname.glob(path + "*.grf").map do |graph_path|
+        #             acc.concat([Graph.create({
+        #               :path => graph_path
+        #             })])
+        #           end
+        #           acc
+        #         end
+        Project.create(project.merge({:graph_repos => repos}))
+      end
+
+      def self.load_sinks(project, path="./sinks/")
+        p = Pathname(path)
+        sinks = Pathname.glob(p + "*.json").map do |p|
+          Sink.create(JSON.parse(File.read(p), :symbolize_names => true))
+        end
+        Project.create(project.merge({:sinks => sinks}))
+      end
+
+      def self.load_flows_definitions(project, path="./flows/")
+        p = Pathname(path)
+        flows = Pathname.glob(p + "*.rb").map do |p|
+          GoodData::Bam::DSL.class_eval(File.read p).to_hash
+        end
+        Project.create(project.merge({:flows_definitions => flows}))
+      end
+
+      def self.build_project(options={})
+        project = Project.create({})
+        paths = options[:graph_repos] || []
+        p = Project.load_taps(project)
+        p = Project.load_sinks(p)
+        p = Project.set_graph_repos(p, paths)
+        p = Project.load_flows_definitions(p)
+        p = Compiler.compile_dsl(p)
+      end
+
+    end
+  end
+end

data/lib/base/repo.rb

@@ -0,0 +1,35 @@
+module GoodData 
+  module Bam
+    module Repository
+      
+      def self.create(repository)
+        fail "repository has to have \"type\" defined. You specified \"#{repository}\"" unless repository.has_key?(:type)
+        case repository[:type]
+        when :file
+          repository.merge({
+            :base => Pathname(repository[:base])})
+        when :hash
+          fail "Repo has to have content defined" if repository[:content].nil? || !repository[:content].is_a?(Hash)
+          repository
+        end
+      end
+      
+      def self.find(repository, path)
+        case repository[:type]
+        when :file
+          path = repository[:base] + Pathname(path)
+          if (path).exist?
+            Graph.create({
+              :path => Pathname(path).expand_path,
+              :repository => repository
+            })
+          end
+        when :hash
+          Graph.create(repository[:content][path])
+        else
+          fail "not implemented"
+        end
+      end
+    end
+  end
+end

data/lib/base/sink.rb

@@ -0,0 +1,44 @@
+module GoodData 
+  module Bam
+    module Sink
+
+      def self.create(sink)
+        fail "Sink should have id. You defined \"#{sink}\"" if sink[:id].blank?
+        fail "Sink should have target. You defined \"#{sink}\"" if sink[:target].blank?
+        sink = {:fields => []}.merge(sink).merge({:type => :sink})
+        add_target_type(sink)
+      end
+
+      def self.find_field(sink, name)
+        sink[:fields].detect {|f| f[:name] == name}
+      end
+
+      def self.has_field?(sink, name)
+        !find_field(sink, name).nil?
+      end
+
+      def self.add_target_type(sink)
+        target = sink[:target].to_s
+        path = Pathname(target)
+        uri = URI(target)
+
+        if target == "gooddata" 
+          sink.merge({
+            :target => :gooddata,
+            :target_type => :gooddata
+          })
+        elsif uri.absolute? && (uri.scheme == "http" || uri.scheme == "https")
+          sink.merge({
+            :target => uri,
+            :target_type => :web
+          })
+        elsif target =~ /^\./ || target =~ /^\//
+          sink.merge({
+            :target => path,
+            :target_type => :file
+          })
+        end
+      end
+    end
+  end
+end

data/lib/base/step.rb

@@ -0,0 +1,47 @@
+module GoodData 
+  module Bam
+    module Step
+
+      def self.get_graph_path(step)
+        case step[:type]
+        when :tap
+          "graphs/#{step[:flow_id]}_#{step[:id]}_#{step[:source_type]}_tap.grf"
+        when :sink
+          "graphs/#{step[:flow_id]}_#{step[:id]}_sink.grf"
+        when :graph
+          Graph.get_path(step)
+        else
+          raise "Not implemented"
+        end
+      end
+
+      def self.has_output_step?(steps)
+        steps.any? {|s| s.has_key?(:out)}
+      end
+
+      def self.find_output_step(steps)
+        if has_output_step?(steps)
+          steps.detect {|s| s.has_key?(:out)}
+        elsif steps.count == 1
+          steps.first
+        else
+          fail "No out is defined and there are more than 1 fields in flow \"#{steps}\""
+        end
+      end
+
+      def self.step_name(step)
+        case step[:type]
+        when :tap
+          "#{step[:id]}_#{step[:type]}"
+        when :sink
+          "#{step[:id]}_#{step[:type]}"
+        when :graph
+          "#{step[:path].basename.to_s.chomp(step[:path].extname)}_#{step[:type]}"
+        else
+          raise "Not implemented"
+        end
+      end
+
+    end
+  end
+end

data/lib/base/tap.rb

@@ -0,0 +1,167 @@
+module GoodData 
+  module Bam
+    
+    module Tap
+
+      def self.create(tap)
+        tap = ({:fields => [], :direct => false}.merge(tap).merge(:type => :tap))
+        tap = Tap.add_source_type(tap)
+        fail "Tap has to have id defined" if tap[:id].nil?
+        
+        tap
+      end
+
+      def self.add_field(tap, field)
+        fail "You have to specify name. You specified \"#{field}\"" unless field.respond_to?(:has_key?) && field.has_key?(:name)
+        fail "There already is a field named #{field[:name]}" if has_field?(tap, field[:name])
+        position = field[:position] || 0
+        fields = tap[:fields].clone
+        fields.insert(position - 1, field)
+        Tap.create(tap.merge({:fields => fields}))
+      end
+
+      def self.remove_field(tap, name)
+        fields = tap[:fields].find_all {|f| f[:name] != name}
+        Tap.create(tap.merge({:fields => fields}))
+      end
+
+      def self.add_source_type(tap)
+        source = tap[:source].to_s
+        path = Pathname(source)
+        uri = URI(source)
+        
+        if uri.absolute? && (uri.scheme == "http" || uri.scheme == "https")
+          tap.merge({
+            :source => uri,
+            :source_type => :web
+          })
+        elsif source =~ /^\./ || source =~ /^\//
+          tap.merge({
+            :source => path,
+            :source_type => :file
+          })
+        elsif source == :dummy
+          tap.merge({
+            :source_type => :dummy
+          })
+        else
+          tap.merge({
+            :source_type => :salesforce
+          })
+        end
+      end
+
+      def self.has_field?(tap, name)
+        !Tap.find_field(tap, name).nil?
+      end
+      
+      def self.find_field(tap, name)
+        tap[:fields].detect {|f| f[:name] == name}
+      end
+      
+      def self.find_output_field(tap, name)
+        acts_as_field = tap[:fields].detect {|f| f[:acts_as] && f[:acts_as].include?(name)}
+        source_field = Tap.find_field(tap, name)
+        if acts_as_field.blank?
+          source_field if source_field && (!source_field.has_key?(:acts_as) || !source_field[:acts_as].include(name))
+        else
+          acts_as_field
+        end
+      end
+      
+      def self.has_output_field?(tap, name)
+        !find_output_field(tap, name).nil?
+      end
+
+      def self.remove_timestamp_field(tap)
+        remove_field(tap, "Timestamp")
+      end
+
+      def self.add_timestamp_field(tap)
+        if has_field?(tap, "Timestamp")
+          tap
+        else
+          add_field(tap, {:name => "Timestamp"})
+        end
+      end
+
+      def self.pull_timestamp_from_output_to_input(tap)
+        if has_field?(tap, "Timestamp")
+          tap
+        elsif has_output_field?(tap, "Timestamp")
+          field = find_output_field(tap, "Timestamp")
+          fields = tap[:fields].clone
+          index = fields.index(field)
+
+          field = field.clone
+          field[:name] = "Timestamp"
+          fields.delete_at(index)
+          fields.insert(index, field)
+          Tap.create(tap.merge(:fields => fields))
+        else
+          tap
+        end
+      end
+
+      def self.pull_id_from_output_to_input(tap)
+        if has_field?(tap, "Id")
+          tap
+        elsif has_output_field?(tap, "Id")
+          field = find_output_field(tap, "Id")
+          fields = tap[:fields].clone
+          index = fields.index(field)
+
+          field = field.clone
+          field[:name] = "Id"
+          fields.delete_at(index)
+          fields.insert(index, field)
+
+          Tap.create(tap.merge(:fields => fields))
+        else
+          tap
+        end
+      end
+
+      def self.source_target_mappging(tap)
+        tap[:fields].reduce([]) do |acc, f|
+          transformation = f[:acts_as] ? f[:acts_as].map {|i| [f[:name], i]} : [[f[:name], f[:name]]]  
+          acc.concat(transformation)
+        end
+      end
+
+      def self.collect_validations(tap)
+        tap[:fields].find_all {|f| f.has_key?(:validates_as)}
+      end
+
+      def self.remove_acts_as(tap)
+        Tap.create(tap.merge({
+          :fields => tap[:fields].map {|f| f = f.clone; f.delete_if {|key, value| key == :acts_as }}
+        }))
+      end
+
+      def self.prepare_for_sf_downloader(tap)
+        id_field = Tap.find_output_field(tap, "Id")
+        time_field = Tap.find_output_field(tap, "Timestamp")
+        tap = Tap.remove_acts_as(tap)
+        fields = tap[:fields].map {|f| f[:name] == id_field[:name] ? f.merge(:acts_as => ["Id"]) : f}
+        if time_field
+          fields = fields.map {|f| f[:name] == time_field[:name] ? f.merge(:acts_as => ["Timestamp"]) : f}
+        end
+        Tap.create(tap.merge({
+          :fields => fields
+        }))
+      end
+
+      def self.prepare_for_es_downloader(tap)
+        tap = Tap.pull_id_from_output_to_input(tap)
+        tap = Tap.remove_timestamp_field(Tap.pull_timestamp_from_output_to_input(tap))
+        tap[:snapshots] == true ? Tap.add_field(tap, {:name => "Snapshot"}) : tap
+      end
+
+      def self.prepare_for_s3_backup(tap)
+        Tap.remove_acts_as(tap)
+      end
+
+    end
+  end
+end

data/lib/base/taps.rb

@@ -0,0 +1,19 @@
+module GoodData 
+  module Bam
+    module Taps
+
+      def self.get_incremental(taps)
+        taps.find_all {|t| t[:direct] == false || t[:direct].nil? }
+      end
+
+      def self.get_salesforce(taps)
+        taps.find_all {|t| t[:source_type] == :salesforce}
+      end
+
+      def self.get_file(taps)
+        taps.find_all {|t| [:web, :file].include?(t[:source_type]) }
+      end
+
+    end
+  end
+end