heydan 0.1.1

data/lib/heydan/script.rb

@@ -0,0 +1,179 @@
+require 'ruby-progressbar'
+class HeyDan::Script
+  attr_accessor :jurisdiction_type
+  attr_accessor :fromsource
+  attr_accessor :data
+  attr_accessor :folder
+  attr_accessor :source
+  attr_accessor :variable
+  attr_accessor :id
+  attr_accessor :source_file
+  attr_accessor :identifiers #builds a hash with new_identifier => file_name
+
+  def initialize(opts)
+    @folder = opts[:folder]
+    @source = opts[:source]
+    @variable  = opts[:variable]
+    @jurisdiction_type = HeyDan.options[:type]
+    @fromsource = HeyDan.options[:fromsource]
+    @identifiers = {}
+    @source_file = HeyDan::SourceFile.new(@folder, @source, @variable)
+  end
+
+  #overwritten by the developer
+  def build
+  end
+
+  #overwritten by the developer
+  def validate_build
+  end
+
+  #overwritten by the developer, can be dataset, attribute, or identifer
+  def type
+    'dataset'
+  end
+
+  #overwritten by developer
+  def version
+    1
+  end
+
+  def dataset_file_name
+    "#{@folder}_#{@source}_#{@variable}.csv"
+  end
+
+  def name
+    "#{@folder}_#{@source}_#{@variable}.csv"
+  end
+
+  def dataset_file_path
+    File.join(HeyDan.folders[:datasets], dataset_file_name)
+  end
+
+  #downloads from the cdn
+  def download
+    @data = HeyDan::Helper.get_data_from_url(HeyDan.cdn + '/' + dataset_file_name)
+  end
+
+  def process_from_source
+    build
+    validate_build
+    HeyDan::Helper.save_data(dataset_file_name, @data)
+  end
+
+  #runs through download, build and validate
+  def process
+    if @fromsource
+      puts "From Source is specified, processing from source for #{name}" if HeyDan.help?
+      process_from_source 
+    end
+
+    begin
+      if HeyDan::Helper.dataset_exists?(dataset_file_name)
+        puts "Dataset for #{name} exists" if HeyDan.help?
+      else
+        download 
+      end
+    rescue 
+        puts "Had trouble downloading #{name}, processing from source instead" if HeyDan.help?
+        process_from_source        
+    end      
+    update_jurisdiction_files
+  end
+
+  def build_identifier_hash(identifier)
+    identifier_file = File.join(HeyDan.folders[:downloads], "identifiers_file_#{identifier}.json")
+    if File.exist?(identifier_file)
+      @identifiers = JSON.parse(File.read(identifier_file))
+      return @identifiers
+    end
+    HeyDan::HelpText.build_identifier(identifier)
+    get_identifiers_from_files(identifier)
+    File.open(identifier_file, 'w') do |file|
+      file.write(@identifiers.to_json)
+    end
+    @identifiers
+  end
+
+  def get_identifiers_from_files(identifier)
+    @identifiers = {} 
+    Dir.glob(File.join(HeyDan.folders[:jurisdictions], '*.json')).each do |j|
+      jf = HeyDan::JurisdictionFile.new(name: j.gsub(HeyDan.folders[:jurisdictions] + '/', ''))
+      return if !jf.match_type?(@jurisdiction_type)
+      @identifiers["#{jf.get_identifier(identifier)}"] = j.gsub(HeyDan.folders[:jurisdictions] + '/', '')
+    end
+    @identifiers
+  end
+
+  def update_jurisdiction_files
+    get_data
+    get_identifiers
+    @progress = ProgressBar.create(:title => "Updating Files for #{@source} #{@variable} from #{@folder} for jurisdictions #{(' matching ' + @jurisdiction_type) if @jurisdiction_type}", :starting_at => 0, :total => @data[1..-1].size) if HeyDan.help?
+    self.send("add_#{type}s")
+    @progress.finish if HeyDan.help?
+  end
+
+  def add_datasets
+    metadata = @source_file.variable
+    metadata.keep_if { |k| ['id', 'name', 'short_description', 'tags'].include?(k)}
+    metadata["years"] = @data[0][1..-1]
+    id = metadata['id']
+    @data[1..-1].each_index do |i| 
+      row = @data[i+1]
+      next if row[0].nil? || @identifiers[row[0]].nil?
+      jf = get_jurisdiction_filename(@identifiers[row[0]])
+      next if row[0].nil? || !jf.exists?
+      ds = jf.get_dataset(id)
+      if !ds.nil?
+        if ds['version'] >= version
+          @progress.progress = i if HeyDan.help?
+          next
+        end
+      end
+      jf.datasets.delete(ds)
+      metadata["version"] = version
+      metadata["data"] = row[1..-1]
+      jf.add_dataset(metadata)
+      jf.save
+      @progress.progress = i if HeyDan.help?
+    end
+  end
+
+  def get_jurisdiction_filename(id)
+    HeyDan::JurisdictionFile.new(name: @identifiers[id] || id)
+  end
+
+  def add_identifiers
+    @data[1..-1].each_index do |i| 
+      row = @data[i+1]
+      jf = get_jurisdiction_filename(row[0])
+      next if row[0].nil? || !jf.exists?
+      jf.add_identifier(@data[0][1], row[1])
+      jf.save
+      @progress.progress = i  if HeyDan.help?
+    end
+  end
+
+  def add_attributes
+    @data[1..-1].each_index do |i| 
+        row = @data[i+1]
+        jf = get_jurisdiction_filename(row[0])
+        next if row[0].nil? || !jf.exists?
+        jf.add_attribute(@data[0][1], row[1])
+        jf.save
+        @progress.progress = i  if HeyDan.help?
+      end
+  end
+
+  def get_identifiers
+    @id = @data[0][0]
+    @identifiers = build_identifier_hash(@id)
+  end
+
+  def get_data
+    if @data.nil?
+      @data = HeyDan::Helper.get_data(dataset_file_name)
+    end
+  end
+
+end

data/lib/heydan/script_file.rb

@@ -0,0 +1,44 @@
+class HeyDan::ScriptFile
+  attr_accessor :folder
+  attr_accessor :source
+  attr_accessor :variable
+  attr_accessor :script_folder_path
+  attr_accessor :script_file_path
+  attr_accessor :class_name
+  attr_accessor :name
+
+
+  def initialize(folder, source, variable)
+    @folder, @source, @variable = folder, source, variable
+    @name = @folder + "_" + @source + "_" + @variable
+    create_class_name
+    @script_folder_path = File.join(HeyDan.folders[:sources], @folder, 'scripts')
+    @script_file_path = File.join(@script_folder_path, "#{@name}.rb")
+  end
+
+  def create_script_folder
+    FileUtils.mkdir_p @script_folder_path if !Dir.exist?(@script_folder_path)
+  end
+
+  def create_class_name
+    @class_name = @name.split('_').collect(&:capitalize).join
+  end
+
+  def template
+    template_path = File.join('lib', 'templates', 'script.rb.erb')
+    require 'erb'
+    ERB.new(File.read(template_path)).result binding
+  end
+
+  def save
+    create_script_folder
+    File.open(@script_file_path, 'w') do |f|
+      f.write(template)
+    end
+  end
+
+  def eval_class
+    load script_file_path
+    return eval("#{@class_name}")
+  end
+end

data/lib/heydan/server.rb

@@ -0,0 +1,55 @@
+require 'sinatra'
+require 'json'
+require 'sinatra/cross_origin'
+require 'elasticsearch'
+
+class HeyDan::Server < Sinatra::Base
+
+  configure do
+    enable :cross_origin
+  end
+
+  get '/entities/*.json' do
+    content_type 'application/json'
+    begin
+      file = File.join(HeyDan.folders[:jurisdictions], "#{params[:splat][0].gsub('/','::')}.json")
+      File.read(file)
+    rescue
+       halt 404, "{\"message\": \"I couldn\'t find #{params[:splat][0]}. Is the ID right?\"}"
+    end
+  end
+
+  get '/entities' do
+    content_type 'application/json'
+    begin
+      client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
+      r = client.search index: 'jurisdictions',type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
+                  body: {
+                    from: params[:page] || 1,
+                    size: params[:per_page] || 10
+                  }
+       return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
+    rescue
+      halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
+    end
+  end
+
+  get '/entities/*' do
+    content_type 'application/json'
+    begin
+      client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
+      r = client.search index: 'jurisdictions', type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
+                  body: {
+                    query: {
+                      simple_query_string:{query: "\"#{params[:splat][0]}*\"", fields: ['id']}
+                    },
+                    from: params[:page] || 1,
+                    size: params[:per_page] || 10
+                  }
+                  return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
+    rescue
+      halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
+    end
+  end
+
+end

data/lib/heydan/source_file.rb

@@ -0,0 +1,79 @@
+require 'json'
+
+class HeyDan::SourceFile
+  attr_accessor :json
+  attr_accessor :file_path
+  attr_accessor :folder_path
+  attr_accessor :folder
+  attr_accessor :name
+  attr_accessor :variable
+
+  def initialize(folder, name, variable=nil)
+    @folder = folder
+    @name = name
+    @variable = variable
+    @folder_path = File.join(HeyDan.folders[:sources],@folder)
+    @file_path = File.join(@folder_path, @name)
+    get_json
+  end
+
+  def get_json
+    if exist?
+      @json = JSON.parse(File.read(file_path))
+    else
+      @json = initial_json
+    end
+  end
+
+  def create_folder
+    FileUtils.mkdir_p @folder_path if !Dir.exist?(@folder_path)
+  end  
+
+  def exist?
+    File.exist?(@file_path)
+  end
+
+  def add_variable(variable_name)    
+    @json['variables'][variable_name] = variable_json(variable_name) if variable(variable_name).nil?
+    variable(variable_name)
+  end
+
+  def create_variable_scripts
+    if @json['variables'].keys.size > 0
+      @json['variables'].keys.each do |variable|
+        create_script_file(variable)
+      end
+    end
+  end
+
+  def create_script_file(variable_name)
+    file = HeyDan::ScriptFile.new(@folder, @name, variable_name)
+    file.save
+  end
+
+  def initial_json
+    {'name' => @name, 'short_description' => 'A short description', 'long_description' => 'a longer description', 'notes' => nil, 'depends' => nil, 'sourceUrl' => 'the website of the source', 'variables' => {}}
+  end
+
+  def variable_json(variable_name)
+    {'id' => "#{@folder}_#{@name}_#{variable_name}",'name' => variable_name, 'short_description' => 'a short description', 'long_description' => 'a description of the variable', 'notes' => 'any notes about this variable', 'identifier' => 'open_civic_id or ansi_id or other', 'dates' => [2015], 'tags' => [], 'sourceUrl' => 'website for variables information if different than source', 'jurisdiction_types' => [], 'coverage' => {} }
+  end
+
+  def variable(variable_name=nil)
+    variable_name ||= @variable 
+    @json['variables'][variable_name]
+  end
+
+  def variables
+    @json['variables'].keys
+  end
+
+  def save
+    create_folder
+    create_variable_scripts
+    File.open(@file_path, 'w') do |f|
+      f.write(JSON.pretty_generate(@json))
+    end
+  end
+
+end

data/lib/heydan/sources.rb

@@ -0,0 +1,128 @@
+require 'git'
+
+class HeyDan::Sources
+  class << self
+    
+    def source_exists?(name_or_link)
+      HeyDan.sources ||= {} 
+      HeyDan.sources.keys.map(&:to_s).include?(name_or_link) || HeyDan.sources.values.include?(name_or_link)
+    end
+
+    def sync
+      keys = HeyDan.sources.keys
+      if keys
+        HeyDan.sources.keys.map(&:to_s).each { |source| update(source)}
+      end
+    end
+
+    def add(link)
+      raise 'Link must be a git link in the format of http(s)://url/*.git' if !correct_link?(link)
+      settings_file = HeyDan::Base.load_or_create_settings
+      name = extract_name(link)
+      if !source_exists?(link)
+        HeyDan.sources ||= {} 
+        HeyDan.sources.merge!({"#{name}" => link})
+        HeyDan::Base.save_settings
+      end
+      update(name)
+    end
+
+    def correct_link?(link)
+      !link.match(/(http|https):\/\/\w+\.\w+(\/\w+)+\.git/).nil?
+    end
+
+    def update(name)
+      if directory_exist?(name)
+        HeyDan::HelpText.git_update(name)
+        g = Git.open(source_folder(name))
+        g.pull
+      else
+        HeyDan::HelpText.git_clone(name)
+        g = Git.clone(HeyDan.sources[name.to_sym], name, {:path => HeyDan.folders[:sources]})
+      end
+    end
+
+    def source_folder(name)
+      File.join(HeyDan.folders[:sources],name)
+    end
+    
+    def directory_exist?(name)
+      Dir.exists?(source_folder(name))
+    end    
+
+    def extract_name(git_link)
+      git_link.match(/(\w+)\.git$/i)[1]
+    end
+
+    def create_folder(name)
+      FileUtils.mkdir_p source_folder(name)
+    end
+
+    def create_source(folder, name)
+      file = HeyDan::SourceFile.new(folder, name)
+      file.save
+    end
+
+    def create_variable(folder, name, variable)
+      file = HeyDan::SourceFile.new(folder, name)
+      file.add_variable(variable)
+      file.save
+    end
+
+    def source_exist?(folder, source_name)
+      file = HeyDan::SourceFile.new(folder, source_name)
+      file.exist?
+    end
+
+    def variable_exist?(folder, source_name, variable)
+      file = HeyDan::SourceFile.new(folder, source_name)
+      return file.exist? if !file.exist?
+      !file.variable(variable).nil?
+    end
+
+    def only_letters_and_underscores?(text)
+    end
+
+    def create(folder, source_name, variable=nil)
+      create_folder(folder) if !directory_exist?(folder)
+      create_source(folder, source_name) if !source_exist?(folder, source_name)
+      create_variable(folder, source_name, variable) if !variable_exist?(folder, source_name, variable)
+    end
+
+    def build(folder=nil, name=nil, variable=nil)
+      if variable && name && folder
+        build_variable(folder, name, variable)
+      elsif name && folder
+        build_source(folder, name)
+      elsif folder
+        build_folder(folder)
+      else
+        HeyDan.sources.keys.each { |source| build_folder(source.to_s)}
+      end
+    end
+
+    def sources(folder)
+      Dir.glob(source_folder(folder) + '/*').select { |x| !x.include?('/scripts') && !x.include?('.')}.map { |x| x.split('/')[-1]}
+    end
+
+    def build_folder(folder)
+      sources(folder).each do |source|
+        build_source(folder, source)
+      end
+    end
+
+    def build_source(folder, source)
+      source_file = HeyDan::SourceFile.new(folder, source)
+      source_file.variables.each do |var|
+        build_variable(folder, source, var)
+      end
+    end
+    
+    def build_variable(folder, source, variable)
+      script = HeyDan::ScriptFile.new(folder, source, variable)
+      var = script.eval_class.send(:new, {folder: folder, source: source, variable: variable}.merge(HeyDan.options))
+      var.process
+    end
+
+  end
+end