wonderdog 0.1.1 → 0.2.0

data/test/esbackup_stripped.rb

@@ -0,0 +1,153 @@
+#!/usr/bin/env ruby
+
+# Simple script to dump elasticsearch indexes as raw JSON
+
+require 'tire'
+require 'zlib'
+require 'socket'
+require 'pathname'
+require 'multi_json'
+
+class ESBackup
+
+  def initialize(output_dir, options = {})
+    Tire::Configuration.url "http://#{options[:host]}:#{options[:port]}"
+    @output_dir   = output_dir || ''
+    @index        = options[:index]
+    @batch_size   = options[:batch_size].to_i
+    @mapping_file = options[:mappings]
+    if options[:query].nil?
+      @query      = nil
+    else
+      @query      = MultiJson.load(options[:query]) rescue nil
+    end
+    if options[:dump_file].nil?
+      @dump_file  = @index
+    else
+      @dump_file  = options[:dump_file]
+    end
+  end
+
+  def dump_mapping
+    index = Tire::Index.new @index
+    File.open(@mapping_file, 'w'){ |f| f.puts index.mapping.to_json }
+  end
+
+  def fullpath dir
+    basedir = dir.start_with?('/') ? dir : File.join(Dir.pwd, dir)
+    FileUtils.mkdir_p(basedir)
+    basedir
+  end
+
+  def gz_output
+    File.join(fullpath(@output_dir), @index + '.gz')
+  end
+
+  def create_scanner
+    scan_opts = { size: @batch_size }
+    additional_query = @query
+    Tire::Search::Scan.new(@index, scan_opts) do
+      # This is fucking stupid; why people have to be cute and make everything DSL only
+      # I'll never understand, but the person who wrote this gem has forced us to ONLY be able to
+      # ask queries in this manner.
+      query do
+        additional_query.each_pair do |key, vals|
+          case vals
+            # Assuming here that you are only asking for one field at a time...this is getting hacky fast
+            when Hash  then self.send(key.to_sym, *vals.to_a.flatten)
+            when Array then self.send(key.to_sym, *vals)
+          end
+        end
+      end if additional_query
+    end
+  end
+
+  def run
+    dump_mapping if @mapping_file
+    gz = Zlib::GzipWriter.open gz_output
+    count = 0
+    create_scanner.each do |document|
+      document.each do |record|
+        json_doc = record.to_hash.except(:type, :_index, :_explanation, :_score, :_version, :highlight, :sort).to_json
+        gz.puts json_doc
+        count += 1
+      end
+    end
+    gz.close
+    puts "#{@index} backup complete. #{count} records written"
+  end
+end
+
+class ESRestore
+
+  def initialize(input, options = {})
+    Tire::Configuration.url "http://#{options[:host]}:#{options[:port]}"
+    @index        = options[:index]
+    @batch_size   = options[:batch_size].to_i
+    @gz_input     = Zlib::GzipReader.open(input)
+    @mapping_file = options[:mappings]
+  end
+
+  def create_index
+    index   = Tire::Index.new @index
+    options = @mapping_file ? { mappings: MultiJson.load(File.read(@mapping_file)) } : {}
+    index.create(options) unless index.exists?
+    index
+  end
+
+  def run
+    reindex = create_index
+    count, documents = 0, []
+    @gz_input.each_line do |json|
+      documents << MultiJson.load(json)
+      count     += 1
+      if count % @batch_size == 0
+        reindex.bulk_create documents
+        puts "#{count} records loaded"
+        documents.clear
+      end
+    end
+    @gz_input.close()
+    reindex.bulk_create documents if not documents.empty?
+    puts "#{@index} restore complete with #{count} records loaded"
+  end
+end
+
+class ESDup
+
+  def initialize(input, options = {})
+    Tire::Configuration.url "http://#{options[:host]}:#{options[:port]}"
+    @index        = options[:index]
+    @batch_size   = options[:batch_size].to_i
+    @gz_input     = Zlib::GzipReader.open(input)
+    @mapping_file = options[:mappings]
+  end
+
+  def create_index
+    index   = Tire::Index.new @index
+    options = @mapping_file ? { mappings: MultiJson.load(File.read(@mapping_file)) } : {}
+    index.create(options) unless index.exists?
+    index
+  end
+
+  def run
+    reindex = create_index
+    count, documents = 0, []
+    @gz_input.each_line do |json|
+      line = MultiJson.load(json)
+      line.delete("_id")
+      line.delete("id")
+      documents << line
+      count     += 1
+      if count % @batch_size == 0
+        reindex.bulk_create documents
+        puts "#{count} records loaded"
+        documents.clear
+      end
+    end
+    @gz_input.close()
+    reindex.bulk_create documents if not documents.empty?
+    puts "#{@index} restore complete with #{count} records loaded"
+  end
+end
+

data/test/mapping.yml

@@ -0,0 +1,327 @@
+ad_tag_count:
+  _all:
+    enabled: false
+  _routing:
+    path: ad_tag_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    ad_tag_id:
+      type: integer
+    creative_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+    seconds:
+      type: integer
+flight_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    flight_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+    seconds:
+      type: integer
+    metric_feature:
+      index: not_analyzed
+      type: string
+creative_count:
+  _all:
+    enabled: false
+  _routing:
+    path: creative_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    creative_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+    seconds:
+      type: integer
+item_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    item_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    flight_id:
+      type: integer
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+signal_value_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    signal_value_id:
+      index: not_analyzed
+      type: string
+    flight_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+placement_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    ad_tag_id:
+      type: integer
+    flight_id:
+      type: integer
+    ext_pl_id:
+      index: not_analyzed
+      type: string
+    creative_id:
+      type: integer
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      index: not_analyzed
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    detail:
+      index: not_analyzed
+      type: string
+    pl_composite:
+      index: not_analyzed
+      type: string
+site_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    ad_tag_id:
+      type: integer
+    flight_id:
+      type: integer
+    ext_site_id:
+      index: not_analyzed
+      type: string
+    creative_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    metric:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    site_composite:
+      index: not_analyzed
+      type: string
+interaction_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    ad_tag_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    flight_id:
+      type: integer
+    creative_id:
+      type: integer
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+    base_feature:
+      index: not_analyzed
+      type: string
+    detail:
+      index: not_analyzed
+      type: string
+    datax:
+      index: not_analyzed
+      type: string
+browser_count:
+  _all:
+    enabled: false
+  _routing:
+    path: flight_id
+  dynamic: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    cnt:
+      type: integer
+    flight_id:
+      type: integer
+    pl_test:
+      index: not_analyzed
+      type: string
+    browser_ua:
+      index: not_analyzed
+      type: string
+    tb_h:
+      type: date
+    feature:
+      index: not_analyzed
+      type: string
+parse_exception:
+  dynamic: false
+  _all:
+    enabled: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    ad_tag_id:
+      type: integer
+    cnt:
+      type: integer
+    error_code:
+      type: string
+      index: not_analyzed
+    tb_h:
+      type: date
+mountweasel:
+  dynamic: false
+  _all:
+    enabled: false
+  properties:
+    _state:
+      type: string
+      index: "no"
+      store: "yes"
+    tb_h:
+      type: date
+    cnt:
+      type: integer