heydan 0.1.1

data/lib/heydan/cli.rb

@@ -0,0 +1,80 @@
+require 'thor'
+
+module HeyDan
+
+  class Source < Thor
+
+    desc "sync", "sync your sources folders from the settings file"
+    def sync()
+      HeyDan::helper_text('sources_sync')
+      HeyDan::Sources.sync
+    end
+
+    desc "add GITHUB_LINK", "Add a new folder of sources from github"
+    def add(github_link)
+      HeyDan::helper_text('sources_add')
+      HeyDan::Sources.add(github_link)
+    end
+
+    desc "update NAME", "update a folder of sources"
+    def update(name)
+      HeyDan::helper_text('sources_update')
+      HeyDan::Sources.update(name)
+    end
+
+    desc "new FOLDER SOURCE VARIABLE", "adds a new source NAME in the FOLDER, and an optional VARIABLE"
+    def new(folder, name, variable=nil)
+      HeyDan::helper_text('sources_new')
+      HeyDan::Sources.create(folder, name, variable)
+    end
+
+    option :fromsource, type: :boolean
+    desc "build FOLDER NAME VARIABLE", "builds a source's variables in FOLDER, or optional VARIABLE. You can pass --type school_district for a specific jurisdiction type, or --from-source to build original files"
+    def build(folder=nil, name=nil, variable=nil)
+      HeyDan::helper_text('sources_build')
+      HeyDan.options = options
+      HeyDan::Sources.build(folder, name, variable)
+    end
+  end
+
+  class Cli < Thor
+    class_option 'type'
+
+    desc "setup DIR", "Setups HeyDan in the current directory or specified DIR"
+    def setup(dir=nil)
+      HeyDan::helper_text('setup')
+      HeyDan::Base.setup(dir)
+    end
+
+    desc "build", "Builds jurisdiction files"
+    def build()
+      HeyDan::helper_text('build')
+      HeyDan.options = options
+      HeyDan::OpenCivicIdentifiers.build
+    end
+    desc 'import', "Imports files into elasticsearch"
+    def import()
+      HeyDan::helper_text('import')
+      HeyDan::Import.process
+    end
+    
+    desc "sources SUBCOMMAND ...ARGS", "manage sources"
+    subcommand "sources", Source
+    
+    desc "server", "starts up the webserver for heydan"
+    def server()
+      puts "Serving up some HeyDan Realness"
+      HeyDan::Server.run!
+    end  
+
+    desc "upload", "starts up the webserver for heydan"
+    def upload()
+      puts "Serving up some HeyDan Realness"
+      HeyDan::Cdn.upload
+    end  
+
+  end
+
+    
+
+end

data/lib/heydan/help_text.rb

@@ -0,0 +1,83 @@
+class HeyDan::HelpText
+  class << self
+    def setup(opts={}) 
+      return if !HeyDan.help?
+      puts %Q(
+          Hi! Adding a jurisdictions, datasets, downloads and sources directory and a settings.yml file. If you want to move these directories to other places, just update their locations in the settings file. 
+
+          If you want to run heydan from a different folder than the settings.yml, create an environment variable:
+
+          export HEYDAN_SETTINGS = full/path/to/settings.yml
+
+          To turn off this help, run 'heydan help off' or set the help in settings to false.
+
+          heydan grabs datasets and information about jurisdictions. If you want to focus on just one type of jurisdiction, update the settings 'jurisdiction_type'. Or you can pass --type school_district to any heydan command.
+
+          Next, run `heydan build` to setup your files.
+      )
+  end
+
+  def build(opts={})
+    return if !HeyDan.help?
+    type = opts[:type] || 'all'
+    puts %Q(
+      Woot, building files for type #{type} jurisdictions/. You will see a progress bar below. If you didn't specify a type, it might take a while. 
+
+      heydan uses the Open Civic Identifiers format to structure file names and main identification for jurisdictions. This helps create a unique nonchanging identification code for every jurisdiction, based on the sponsoring parent. So, the State of Kansas, would be country:us/state:kansas. heydan creates a flat json file for each jurisdiction, which you can then import into your own application or elasticsearch.
+
+      Next, run heydan sources sync
+      )
+  end
+
+  def sources_add
+    return if !HeyDan.help?
+    puts %Q(
+      You can leverage the community of developers out there. Add the github link to a source repo and tap into all that hardwork.
+
+      When you add a new one, it will get stored in your settings file under 'sources'
+      )
+  end
+
+  def sources_sync
+    return if !HeyDan.help?
+    puts %Q(
+      Sync all the sources in your settings file.
+      )
+  end
+
+  def sources_update
+    return if !HeyDan.help?
+    puts %Q(
+      Update a single source.
+      )
+  end
+
+  def sources_build
+  end
+
+  def sources_new
+  end
+
+  def git_clone(name)
+    return if !HeyDan.help?
+    puts %Q(Cloning #{name} into #{HeyDan.folders[:sources]})
+  end
+
+  def git_update(name)
+    return if !HeyDan.help?
+    puts %Q(Updating #{name} in #{HeyDan.folders[:sources]})
+  end
+
+  def build_identifier(identifier)
+    return if !HeyDan.help?
+    puts %Q("building identifiers hash for #{identifier} to filenames, this might take a moment")
+  end
+
+  def import
+    return if !HeyDan.help?
+    puts "This will import all the files/contents from the jurisdictions folder into ElasticSearch. Make sure it's running!"
+  end
+
+end
+
+end

data/lib/heydan/helper.rb

@@ -0,0 +1,108 @@
+require 'digest'
+require 'uri'
+require 'open-uri'
+require 'csv'
+
+class HeyDan::Helper
+
+  class << self
+
+    def classify(name)
+      name.split('_').collect(&:capitalize).join
+    end
+
+    def download(url)
+      path = HeyDan.folders[:downloads]
+      new_file = File.join(path, md5_name(url))
+      return new_file if File.exist?(new_file)
+      download_file(url, new_file)
+      new_file
+    end
+
+    def save_data(name, data)
+      full_path = File.expand_path(File.join(HeyDan.folders[:datasets], "#{name.gsub('.csv', '')}.csv"))
+      CSV.open(full_path, 'w') do |csv|
+        data.each do |row|
+          csv << row
+        end
+      end
+    end
+
+    def get_data(name)
+      CSV.read(File.join(HeyDan.folders[:datasets], "#{name.gsub('.csv', '')}.csv"))
+    end
+
+    def dataset_exists?(name)
+      File.exist?(File.join(HeyDan.folders[:datasets], "#{name.gsub('.csv', '')}.csv"))
+    end
+
+    def get_data_from_url(url)
+      ext = get_file_type_from_url(url)
+      file = download(url)
+      @data = case ext
+        when 'csv'
+          get_csv_data(file)        
+        when 'zip'
+          files = unzip(file)
+          if files.size == 1
+            get_csv_data(files[0]) if is_csv?(files[0])
+          else
+            files.map { |f| get_csv_data(f) if is_csv?(f)} 
+          end
+        when 'txt'
+          get_csv_data(file) if is_csv?(file)
+        else
+          get_csv_data(file) if is_csv?(file)
+        end
+      @data
+    end
+
+    def is_csv?(file_path)
+      contents = File.open(file_path, &:readline)
+      contents.match(/\b\t/).nil? || contents.match(/\b,/).nil? #not perfect
+    end
+
+    def get_csv_data(file)
+      contents = File.read(file, :encoding => 'utf-8').encode("UTF-8", :invalid=>:replace, :replace=>"").gsub('"',"")
+
+      if contents.include?("\t")
+        CSV.parse(contents, { :col_sep => "\t" })
+      else
+        CSV.parse(contents)
+      end
+      
+    end
+
+    def md5_name(text)
+      Digest::MD5.hexdigest(text)
+    end
+
+    def download_file(url,file_path)
+      f = open(url)
+      full_path = File.expand_path(file_path)
+      File.open(full_path, 'wb') do |saved_file|
+        saved_file.write(f.read)
+      end 
+      full_path
+    end
+
+    def get_file_type_from_url(url)
+      file_type = File.extname(URI.parse(url).path).gsub('.', '') 
+    end
+
+    def unzip(file)
+      path = HeyDan.folders[:downloads]
+      require 'zip'
+      files = []
+      Zip::File.open(file) do |zip_file|
+        zip_file.each do |entry|
+          download_path = File.expand_path(File.join(path, entry.name))
+          entry.extract(download_path) unless File.exists?(download_path)
+          files << download_path
+        end
+      end
+      files
+    end
+
+  end
+end

data/lib/heydan/import.rb

@@ -0,0 +1,49 @@
+require 'ruby-progressbar'
+require 'elasticsearch'
+
+class HeyDan::Import
+    attr_accessor :client
+
+    class << self
+      def client
+        @client ||= Elasticsearch::Client.new host: HeyDan.elasticsearch[:url], log: false
+      end
+
+      def index
+        @index ||= 'jurisdictions'
+      end
+
+      def check_index
+        client.indices.exists? index: index
+      end
+
+      def create_index
+        client.indices.create index: index
+      end
+
+      def process
+        create_index unless check_index
+        total = Dir.glob("#{HeyDan.folders[:jurisdictions]}/*").size
+        files= Dir.glob("#{HeyDan.folders[:jurisdictions]}/*")
+        a=0
+        b=10000
+        progress = ProgressBar.create(:title => "Importing #{files.size} jurisdictions into Elastic Search", :starting_at => a, :total => files.size)
+        while true do
+          @bulk = []
+          b=( files.size - b < 10000 ? -1 : a + 10000)
+          files[a..b].each do |file|
+            jf = HeyDan::JurisdictionFile.new(name: file)
+            @bulk << { index:  { _index: 'jurisdictions', _type: jf.type, _id: jf.hash_id, data: jf.get_json } } 
+          end
+          @client.bulk refresh: true, body: @bulk; nil    
+          a = b + 1
+          if b == -1
+            progress.finish
+            break 
+          else
+            progress.progress = a 
+          end
+        end
+      end
+    end
+end

data/lib/heydan/jurisdiction_file.rb

@@ -0,0 +1,119 @@
+require 'json'
+
+class HeyDan::JurisdictionFile
+  attr_accessor :name
+  attr_accessor :json
+  
+  def initialize(opts={})
+    @name = opts[:name]
+    convert_file_name if @name.include?('.json')
+    @name = @name.gsub('jurisdictions/','').gsub('ocd-division/','').gsub(/\.\.\//,'')
+    raise "Name is required" if @name.nil?
+  end
+
+  def type
+    @name.split('/')[-1].split(':')[0]
+  end
+
+  def id
+    get_json
+    @json['id']
+  end
+
+  def match_type?(ocd_type)
+    return true if ocd_type.nil?
+    !id.match(/#{ocd_type.gsub(':all', '.+')}/).nil?
+  end
+
+  def convert_file_name
+    @name = @name.split('/')[-1]
+    @name = "#{@name.gsub('::','/').gsub('.json','')}"
+  end
+
+  def file_name
+    "#{@name.gsub(/\//, '::').gsub('ocd-division::', '')}.json"
+  end
+
+  def hash_id
+    HeyDan::Helper.md5_name(@name.gsub(/\.\.\//,''))
+  end
+
+  def exists?
+    File.exists?(file_path)
+  end
+
+  def file_path
+    File.expand_path(File.join(HeyDan.folders[:jurisdictions], file_name))
+  end
+
+  def type
+    @name.split('/')[-1].split(':')[0]
+  end
+
+  def initial_json
+    {'id' => @name, 'entityType' => type, 'attributes'=> {}, 'identifiers' => {}, 'datasets' => []}
+  end
+
+  def get_identifier(key)
+    get_json
+    @json['identifiers'][key]
+  end
+
+  def add_identifier(key, value)
+    get_json
+    @json['identifiers'][key] = value
+    @json
+  end
+
+  def add_dataset(value)
+    get_json
+    @json['datasets'] << value 
+    @json
+  end
+
+  def get_dataset(key)
+    get_json
+    @json['datasets'].select { |d| d['id']==key}[0]
+  end
+
+  def datasets
+    get_json
+    @json['datasets']
+  end
+
+  def add_property(key, value)
+    return false if ['datasets', 'identifiers', 'id', 'entityType', 'attributes'].include?(key)
+     get_json
+    @json[key] = value
+    @json
+  end
+
+  def add_attribute(key, value)
+     get_json
+    @json['attributes'][key] = value
+    @json
+  end
+
+  def get_attribute(key)
+     get_json
+    @json['attributes'][key]
+  end
+
+  def get_json
+    if !exists?  
+      @json ||= initial_json 
+    else
+      file = File.read(file_path)
+      @json ||= initial_json if file == ""
+    end
+    return @json if @json
+    @json = JSON.parse(file)
+  end
+
+  def save
+    File.open(file_path, 'w') do |f|
+      f.write(@json.to_json)
+    end
+  end
+
+end

data/lib/heydan/open_civic_identifiers.rb

@@ -0,0 +1,51 @@
+require 'ruby-progressbar'
+
+class HeyDan::OpenCivicIdentifiers 
+  include HeyDan
+
+  class << self
+    attr_accessor :jurisdiction_type
+    attr_accessor :jurisdictions_folder
+    attr_accessor :data
+
+    def name
+      'open_civic_data'
+    end
+
+    def build(opts={})
+      @jurisdiction_type = HeyDan.options[:type]
+      HeyDan::Base.load_or_create_settings
+      HeyDan::Base.create_folders
+      @jurisdictions_folder = HeyDan.folders[:jurisdictions]
+      download
+      build_jurisdiction_files
+    end
+
+    def download
+      if !HeyDan::Helper.dataset_exists?(name)
+        @data = HeyDan::Helper.get_data_from_url('https://github.com/opencivicdata/ocd-division-ids/blob/master/identifiers/country-us.csv?raw=true')
+        @data = @data[1..-1].map { |c| [c[0], c[1]]}
+        @data.unshift(['id', 'name'])
+        HeyDan::Helper.save_data(name, @data)
+      end
+    end
+
+    def build_jurisdiction_files
+      if @data.nil?
+        @data = HeyDan::Helper.get_data(name)
+      end
+      @progress = ProgressBar.create(:title => "Building Files in #{HeyDan.folders[:jurisdictions]} for jurisdictions #{('matching ' + @jurisdiction_type) if @jurisdiction_type}", :starting_at => 0, :total => @data[1..-1].size) if HeyDan.help?
+      @data[1..-1].each_index do |i| 
+        row = @data[i+1]
+        jf = HeyDan::JurisdictionFile.new(name: row[0])
+        next if !jf.match_type?(@jurisdiction_type)
+        jf.add_identifier('open_civic_id', row[0].gsub('ocd-division/',''))
+        jf.add_property('name', row[1])
+        jf.save
+        @progress.progress = i  if HeyDan.help?
+      end
+      @progress.finish if HeyDan.help?
+    end
+  end
+
+end