wukong-migrate 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ YTc5OTFjNjUwMzI5NzA2ODViNzZjMTBkYWY4YWE0YTg3Y2I0ZTEzZA==
5
+ data.tar.gz: !binary |-
6
+ ZDQ3MzMxMjM0MjUzOWI4NzcyYTg4NmFmZmI4NTVmZjYwNmNhNTA0ZQ==
7
+ !binary "U0hBNTEy":
8
+ metadata.gz: !binary |-
9
+ ZmY2YTllNDBjY2E5ZjNhYjg2MzA3OGQ0ZTg2YTJkNTQzZDNkMDNiYzQ0MmE5
10
+ N2Y5OWMxZjE0ZDFiNDhhOTkxYTljNTdmZmUwNWVkZDg2N2ViM2Q4Mzk2YmU4
11
+ YTIxZDljNjJiMTE2YmUzOWE1OWQwY2NhNjQ3ZTQ3MGMyMjk4ZGM=
12
+ data.tar.gz: !binary |-
13
+ ZjI0MDM3OGJkYzM1YTliOTUyZjY3NTQ0NDIzZGRmODJiYjE4OThmMDNmZjc0
14
+ YjFjNzEzYmZjZDZhZTBjNGUyMDE2MjU2M2FiMWMyNjFiYWY4MGQ1NzhmYmJj
15
+ NGE4OWUyYzIwMzQ0ZTFmMmRkY2JiZjRmNzBhNmM3NjJjZmM3Njg=
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ group :test do
6
+ gem 'rspec', '~> 2'
7
+ end
data/Gemfile.lock ADDED
@@ -0,0 +1,67 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ wukong-migrate (0.0.1)
5
+ gorillib (~> 0.5)
6
+ httparty (~> 0.11)
7
+ rake (>= 0.8.7)
8
+ wukong-deploy (>= 0.1.1)
9
+
10
+ GEM
11
+ remote: https://rubygems.org/
12
+ specs:
13
+ configliere (0.4.18)
14
+ highline (>= 1.5.2)
15
+ multi_json (>= 1.1)
16
+ diff-lcs (1.2.4)
17
+ diffy (3.0.1)
18
+ erubis (2.7.0)
19
+ eventmachine (1.0.3)
20
+ forgery (0.5.0)
21
+ gorillib (0.5.0)
22
+ configliere (>= 0.4.13)
23
+ json
24
+ multi_json (>= 1.1)
25
+ highline (1.6.19)
26
+ httparty (0.11.0)
27
+ multi_json (~> 1.0)
28
+ multi_xml (>= 0.5.2)
29
+ json (1.8.0)
30
+ log4r (1.1.10)
31
+ multi_json (1.7.7)
32
+ multi_xml (0.5.4)
33
+ rake (0.9.6)
34
+ rspec (2.14.1)
35
+ rspec-core (~> 2.14.0)
36
+ rspec-expectations (~> 2.14.0)
37
+ rspec-mocks (~> 2.14.0)
38
+ rspec-core (2.14.4)
39
+ rspec-expectations (2.14.0)
40
+ diff-lcs (>= 1.1.3, < 2.0)
41
+ rspec-mocks (2.14.1)
42
+ uuidtools (2.1.4)
43
+ vayacondios-client (0.2.6)
44
+ configliere (>= 0.4.16)
45
+ gorillib (>= 0.4.2)
46
+ multi_json (>= 1.3.6)
47
+ wukong (3.0.1)
48
+ configliere (>= 0.4.18)
49
+ eventmachine
50
+ forgery
51
+ gorillib (>= 0.4.2)
52
+ log4r
53
+ multi_json (>= 1.3.6)
54
+ uuidtools
55
+ vayacondios-client (>= 0.1.2)
56
+ wukong-deploy (0.1.1)
57
+ diffy
58
+ erubis
59
+ rake (~> 0.9)
60
+ wukong (= 3.0.1)
61
+
62
+ PLATFORMS
63
+ ruby
64
+
65
+ DEPENDENCIES
66
+ rspec (~> 2)
67
+ wukong-migrate!
data/Rakefile ADDED
@@ -0,0 +1,7 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rspec/core/rake_task'
5
+ RSpec::Core::RakeTask.new(:specs)
6
+
7
+ task :default => [:specs]
data/bin/wu-migrate ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'wukong-migrate'
4
+
5
+ Wukong::Migrate::MigrateRunner.run
@@ -0,0 +1,99 @@
1
+ module Gorillib
2
+ module Builder
3
+
4
+ def getset(field, *args, &block)
5
+ ArgumentError.check_arity!(args, 0..1)
6
+ if args.empty?
7
+ read_attribute(field.name)
8
+ else
9
+ self.send("receive_#{field.name}", args.first)
10
+ end
11
+ end
12
+
13
+ def getset_collection_item(field, item_key, attrs={}, &block)
14
+ plural_name = field.plural_name
15
+ if attrs.is_a?(field.item_type)
16
+ # actual object: assign it into collection
17
+ val = attrs
18
+ set_collection_item(plural_name, item_key, val)
19
+ elsif has_collection_item?(plural_name, item_key)
20
+ # existing item: retrieve it, updating as directed
21
+ val = get_collection_item(plural_name, item_key)
22
+ val.receive!(attrs, &block)
23
+ else
24
+ # missing item: autovivify item and add to collection
25
+ params = { key_method => item_key, :owner => self }.merge(attrs)
26
+ val = self.send("receive_#{field.singular_name}", params, &block)
27
+ set_collection_item(plural_name, item_key, val)
28
+ end
29
+ val
30
+ end
31
+
32
+ GetsetCollectionField.class_eval do
33
+ def inscribe_methods model
34
+ raise "Plural and singular names must differ: #{self.plural_name}" if (singular_name == plural_name)
35
+ #
36
+ @visibilities[:writer] = false
37
+ model.__send__(:define_attribute_reader, self.name, self.type, visibility(:reader))
38
+ model.__send__(:define_attribute_tester, self.name, self.type, visibility(:tester))
39
+ #
40
+ model.__send__(:define_collection_receiver, self)
41
+ model.__send__(:define_collection_getset, self)
42
+ model.__send__(:define_collection_tester, self)
43
+ #
44
+ model.__send__(:define_collection_single_receiver, self)
45
+ end
46
+ end
47
+ end
48
+
49
+ module Model
50
+
51
+ ClassMethods.class_eval do
52
+ def to_mapping
53
+ {
54
+ properties: fields.inject({}) do |mapping, (name, field)|
55
+ info = field.type.respond_to?(:to_mapping) ? field.type.to_mapping : field.to_mapping
56
+ mapping[name] = info
57
+ mapping
58
+ end
59
+ }
60
+ end
61
+
62
+ def define_collection_single_receiver field
63
+ collection_single_field_name = field.singular_name
64
+ field_type = field.item_type
65
+ define_meta_module_method("receive_#{collection_single_field_name}", true) do |attrs, &block|
66
+ begin
67
+ field_type.receive(attrs, &block)
68
+ rescue StandardError => err ; err.polish("#{self.class}.#{field_name} type #{type} on #{val}") rescue nil ; raise ; end
69
+ end
70
+ end
71
+ end
72
+
73
+ Field.class_eval do
74
+ field :es_options, Hash
75
+
76
+ def receive_as_type(factory, params)
77
+ product = factory.try(:product)
78
+ case
79
+ when product == Integer
80
+ EsInteger.receive(params)
81
+ when product == Float
82
+ EsFloat.receive(params)
83
+ when product == Date
84
+ EsDate.receive(params)
85
+ when product == [TrueClass, FalseClass]
86
+ EsBoolean.receive(params)
87
+ when product == Array
88
+ receive_as_type(factory.items_factory, params)
89
+ else
90
+ EsString.receive(params)
91
+ end
92
+ end
93
+
94
+ def to_mapping
95
+ receive_as_type(type, es_options || {}).to_mapping
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,18 @@
1
+ require 'gorillib/builder'
2
+ require 'gorillib/metaprogramming/class_attribute'
3
+ require 'gorillib/model/serialization'
4
+ require 'gorillib/object/try'
5
+ require 'gorillib/string/inflections'
6
+ require 'gorillib/string/constantize'
7
+ require 'gorillib/type/extended'
8
+
9
+ require 'httparty'
10
+
11
+ require 'wukong-deploy'
12
+
13
+ require 'gorillib/model/elasticsearch_ext'
14
+ require 'wukong-migrate/dsl'
15
+ require 'wukong-migrate/elasticsearch_fields'
16
+ require 'wukong-migrate/elasticsearch_operations'
17
+ require 'wukong-migrate/elasticsearch_migration'
18
+ require 'wukong-migrate/migrate_runner'
@@ -0,0 +1,38 @@
1
+ module Wukong
2
+ module Migration
3
+
4
+ Registry = {} unless defined? Registry
5
+
6
+ class << self
7
+ def all_migrations
8
+ Registry.keys
9
+ end
10
+
11
+ def retrieve name
12
+ Registry[name.to_s]
13
+ end
14
+
15
+ def register(name, migration)
16
+ Registry[name.to_s] = migration
17
+ end
18
+ end
19
+
20
+ class Dsl
21
+ include Gorillib::Builder
22
+
23
+ field :name, String
24
+ field :log, Whatever
25
+
26
+ class << self
27
+ def define(name, &operations)
28
+ Wukong::Migration.register(name, self.new(&operations))
29
+ true
30
+ end
31
+
32
+ def perform(options = {})
33
+ end
34
+ end
35
+ end
36
+ end
37
+ end
38
+
@@ -0,0 +1,59 @@
1
+ # -*- coding: utf-8 -*-
2
+ class EsField
3
+ include Gorillib::Model
4
+
5
+ field :index_as, String, doc: 'The name of the field that will be stored in the index. Defaults to the property/field name.'
6
+ field :store, String, doc: 'Set to yes to store actual field in the index, no to not store it. Defaults to no (note, the JSON document itself is stored, and it can be retrieved from it).'
7
+ field :index, :boolean, doc: 'Set to analyzed for the field to be indexed and searchable after being broken down into token using an analyzer. not_analyzed means that its still searchable, but does not go through any analysis process or broken down into tokens. no means that it won’t be searchable at all (as an individual field; it may still be included in _all). Setting to no disables include_in_all. Defaults to analyzed.'
8
+ field :boost, Float, doc: 'The boost value. Defaults to 1.0.'
9
+ field :include_in_all, :boolean, doc: 'Should the field be included in the _all field (if enabled). If index is set to no this defaults to false, otherwise, defaults to true or to the parent object type setting.'
10
+
11
+ def to_mapping
12
+ attributes.compact_blank.merge(type: short_type)
13
+ end
14
+ end
15
+
16
+ class EsString < EsField
17
+ field :index, String, default: 'not_analyzed', doc: 'Set to analyzed for the field to be indexed and searchable after being broken down into token using an analyzer. not_analyzed means that its still searchable, but does not go through any analysis process or broken down into tokens. no means that it won’t be searchable at all (as an individual field; it may still be included in _all). Setting to no disables include_in_all. Defaults to analyzed.'
18
+ field :term_vector, String, doc: 'Possible values are no, yes, with_offsets, with_positions, with_positions_offsets. Defaults to no.'
19
+ field :null_value, String, doc: 'When there is a (JSON) null value for the field, use the null_value as the field value. Defaults to not adding the field at all.'
20
+ field :omit_norms, String, doc: 'Boolean value if norms should be omitted or not. Defaults to false for analyzed fields, and to true for not_analyzed fields.'
21
+ field :index_options, String, doc: 'Allows to set the indexing options, possible values are docs (only doc numbers are indexed), freqs (doc numbers and term frequencies), and positions (doc numbers, term frequencies and positions). Defaults to positions for analyzed fields, and to docs for not_analyzed fields. Since 0.20.'
22
+ field :analyzer, String, doc: 'The analyzer used to analyze the text contents when analyzed during indexing and when searching using a query string. Defaults to the globally configured analyzer.'
23
+ field :index_analyzer, String, doc: 'The analyzer used to analyze the text contents when analyzed during indexing.'
24
+ field :search_analyzer, String, doc: 'The analyzer used to analyze the field when part of a query string. Can be updated on an existing field.'
25
+ field :ignore_above, Integer, doc: 'The analyzer will ignore strings larger than this size. Useful for generic not_analyzed fields that should ignore long text. (since @0.19.9).'
26
+ field :position_offset_gap, Integer, doc: 'Position increment gap between field instances with the same field name. Defaults to 0.'
27
+
28
+ def short_type() 'string' ; end
29
+ end
30
+
31
+ class EsNumeric < EsField
32
+ field :precision_step, Integer, doc: 'The precision step (number of terms generated for each number value). Defaults to 4.'
33
+ field :ignore_malformed, :boolean, doc: 'Ignored a malformed number. Defaults to false. (Since @0.19.9).'
34
+ end
35
+
36
+ class EsInteger < EsNumeric
37
+ field :null_value, Integer, doc: 'When there is a (JSON) null value for the field, use the null_value as the field value. Defaults to not adding the field at all.'
38
+ def short_type() 'integer' ; end
39
+ end
40
+
41
+ class EsFloat < EsNumeric
42
+ field :null_value, Float, doc: 'When there is a (JSON) null value for the field, use the null_value as the field value. Defaults to not adding the field at all.'
43
+ def short_type() 'float' ; end
44
+ end
45
+
46
+ class EsDate < EsNumeric
47
+ field :format, String, doc: 'The date format. Defaults to dateOptionalTime.'
48
+ field :null_value, Date, doc: 'When there is a (JSON) null value for the field, use the null_value as the field value. Defaults to not adding the field at all.'
49
+ def short_type() 'date' ; end
50
+ end
51
+
52
+ class EsBoolean < EsField
53
+ field :null_value, :boolean, doc: 'When there is a (JSON) null value for the field, use the null_value as the field value. Defaults to not adding the field at all.'
54
+ def short_type() 'boolean' ; end
55
+ end
56
+
57
+ class EsIpAddress < EsNumeric
58
+ def short_type() 'ip' ; end
59
+ end
@@ -0,0 +1,155 @@
1
+ class EsMigrationDsl < Wukong::Migration::Dsl
2
+ include EsHttpOperation::Helpers
3
+
4
+ def self.template name
5
+ <<-TEMPLATE.gsub(/^ {6}/, '').strip
6
+ EsMigration.define '#{name}' do
7
+ # Use dsl methods to:
8
+ # * create/update/delete indices
9
+ # * update index settings
10
+ # * add/remove aliases
11
+ # * create/update/delete mappings using models defined in app/models
12
+ #
13
+ # create_index(:index_name) do
14
+ # number_of_replicas 5
15
+ # alias_to [:alias_one, :alias_two]
16
+ # create_mapping(:model_name) do
17
+ # dynamic true
18
+ # ttl true
19
+ # end
20
+ # end
21
+ end
22
+ TEMPLATE
23
+ end
24
+
25
+ def operation_list
26
+ @operation_list ||= []
27
+ end
28
+
29
+ end
30
+
31
+ class ObjectDsl < EsMigrationDsl
32
+ # Additional mapping-level settings
33
+ magic :source, :boolean, doc: 'Should the raw JSON be indexed under _source'
34
+ magic :dynamic, :boolean, doc: 'Should the document schema be dynamic'
35
+ magic :all, :boolean, doc: 'Should the document be indexed in _all'
36
+ magic :timestamp, :boolean, doc: 'Should the _timestamp be indexed'
37
+ magic :ttl, String, doc: 'Enable _ttl with a default time'
38
+ magic :analyzer_field, String, doc: 'Specify a field this document should use as an analyzer'
39
+ magic :boost_field, String, doc: 'Specify a field this document should use as a boost'
40
+ magic :parent, String, doc: 'Specify this documents _parent type'
41
+
42
+ def mapping_rules
43
+ {}.tap do |rules|
44
+ rules[:dynamic] = dynamic if attribute_set?(:dynamic)
45
+ rules[:_all] = { enabled: all } if attribute_set?(:all)
46
+ rules[:_source] = { enabled: source } if attribute_set?(:source)
47
+ rules[:_ttl] = { enabled: true, default: ttl } if attribute_set?(:ttl)
48
+ rules[:_timestamp] = { enabled: timestamp } if attribute_set?(:timestamp)
49
+ rules[:_analyzer] = { path: analyzer_field } if attribute_set?(:analyzer_field)
50
+ rules[:_boost] = { name: boost_field } if attribute_set?(:boost_field)
51
+ rules[:_parent] = { type: parent } if attribute_set?(:parent)
52
+ end
53
+ end
54
+
55
+ def model_mapping
56
+ name.to_s.camelize.constantize.to_mapping
57
+ end
58
+
59
+ def to_mapping
60
+ model_mapping.merge(mapping_rules)
61
+ end
62
+ end
63
+
64
+ class IndexDsl < EsMigrationDsl
65
+ # Dsl methods
66
+ collection :creations, ObjectDsl, singular_name: 'create_mapping'
67
+ collection :updates, ObjectDsl, singular_name: 'update_mapping'
68
+ collection :deletions, ObjectDsl, singular_name: 'delete_mapping'
69
+ magic :alias_to, Array, of: Symbol, default: []
70
+ magic :remove_alias, Array, of: Symbol, default: []
71
+
72
+ # Additional index-level settings
73
+ magic :number_of_replicas, Integer
74
+
75
+ def receive_create_mapping(attrs, &block)
76
+ obj = ObjectDsl.receive(attrs, &block)
77
+ operation_list << update_mapping_op(self.name, obj.name, obj.to_mapping)
78
+ obj
79
+ end
80
+
81
+ def receive_update_mapping(attrs, &block)
82
+ obj = ObjectDsl.receive(attrs, &block)
83
+ operation_list << update_mapping_op(self.name, obj.name, obj.to_mapping)
84
+ obj
85
+ end
86
+
87
+ def receive_delete_mapping(attrs, &block)
88
+ obj = ObjectDsl.receive(attrs, &block)
89
+ operation_list.unshift update_mapping_op(self.name, obj.name, obj.to_mapping)
90
+ obj
91
+ end
92
+
93
+ def receive_alias_to params
94
+ params.each{ |als| operation_list << alias_index_op(:add, self.name, als) }
95
+ super(params)
96
+ end
97
+
98
+ def receive_remove_alias params
99
+ params.each{ |als| operation_list << alias_index_op(:remove, self.name, als) }
100
+ super(params)
101
+ end
102
+
103
+ def index_settings
104
+ { number_of_replicas: number_of_replicas }.compact_blank
105
+ end
106
+
107
+ end
108
+
109
+ class EsMigration < EsMigrationDsl
110
+ collection :creations, IndexDsl, singular_name: 'create_index'
111
+ collection :updates, IndexDsl, singular_name: 'update_index'
112
+ collection :deletions, IndexDsl, singular_name: 'delete_index'
113
+
114
+ def receive_create_index(attrs, &block)
115
+ idx = IndexDsl.receive(attrs, &block)
116
+ operation_list << create_index_op(idx.name, idx.index_settings)
117
+ idx
118
+ end
119
+
120
+ def receive_update_index(attrs, &block)
121
+ idx = IndexDsl.receive(attrs, &block)
122
+ operation_list << update_settings_op(idx.name, idx.index_settings)
123
+ idx
124
+ end
125
+
126
+ def receive_delete_index(attrs, &block)
127
+ idx = IndexDsl.receive(attrs, &block)
128
+ operation_list.unshift delete_index_op(idx.name)
129
+ idx
130
+ end
131
+
132
+ def nested_operations
133
+ (creations.to_a + updates.to_a).map{ |idx| idx.operation_list }.flatten
134
+ end
135
+
136
+ def combined_operations
137
+ operation_list + nested_operations
138
+ end
139
+
140
+ def perform(options = {})
141
+ combined_operations.each do |op|
142
+ op.configure_with options
143
+ log.info op.info
144
+ log.debug op.raw_curl_string
145
+ response = op.execute
146
+ log.debug [response.code, response.parsed_response].join(' ')
147
+ if response.code == 200
148
+ log.info 'Operation complete'
149
+ else
150
+ log.error response.parsed_response
151
+ break unless options[:force]
152
+ end
153
+ end
154
+ end
155
+ end
@@ -0,0 +1,100 @@
1
+ class EsHttpOperation
2
+ include Gorillib::Model
3
+ include HTTParty
4
+
5
+ field :index, String
6
+
7
+ def configure_with options
8
+ uri = [options[:host], options[:port]].join(':')
9
+ self.class.base_uri uri
10
+ end
11
+
12
+ def execute
13
+ response = call_own_http_method
14
+ response
15
+ end
16
+
17
+ def call_own_http_method
18
+ http_options = body ? { body: json_body } : {}
19
+ self.class.send(verb, path, http_options)
20
+ end
21
+
22
+ def raw_curl_string
23
+ "curl -X #{verb.to_s.upcase} '#{self.class.base_uri}#{path}'".tap do |raw|
24
+ raw << " -d '#{json_body}'" if body
25
+ end
26
+ end
27
+
28
+ def json_body
29
+ MultiJson.encode(body)
30
+ end
31
+
32
+ class CreateIndex < EsHttpOperation
33
+ field :settings, Hash
34
+
35
+ def path() File.join('', index, '') ; end
36
+ def body() { settings: settings }.compact_blank ; end
37
+ def verb() :put ; end
38
+ def info() "Creating index #{index}" ; end
39
+ end
40
+
41
+ class DeleteIndex < EsHttpOperation
42
+ field :obj_type, String
43
+
44
+ def path() ['', index, obj_type, ''].compact.join('/') ; end
45
+ def body() nil ; end
46
+ def verb() :delete ; end
47
+ def info() "Deleting index #{index}" ; end
48
+ end
49
+
50
+ class UpdateIndexSettings < EsHttpOperation
51
+ field :settings, Hash
52
+
53
+ def path() File.join('', index, '_settings?') ; end
54
+ def body() { index: settings } ; end
55
+ def verb() :put ; end
56
+ def info() "Updating settings for index #{index}" ; end
57
+ end
58
+
59
+ class AliasIndex < EsHttpOperation
60
+ field :alias_name, String
61
+ field :action, Symbol
62
+
63
+ def path() '/_aliases?' ; end
64
+ def body() { actions: [{ action => { index: index, alias: alias_name } }]} ; end
65
+ def verb() :post ; end
66
+ def info() "#{action.capitalize} alias :#{alias_name} for index #{index}" ; end
67
+ end
68
+
69
+ class UpdateIndexMapping < EsHttpOperation
70
+ field :obj_type, String
71
+ field :mapping, Hash
72
+
73
+ def path() File.join('', index, obj_type, '_mapping?') ; end
74
+ def body() { obj_type => mapping } ; end
75
+ def verb() :put ; end
76
+ def info() "Updating #{obj_type} mapping for index #{index}" ; end
77
+ end
78
+
79
+ module Helpers
80
+ def create_index_op(index, settings)
81
+ CreateIndex.receive(index: index, settings: settings)
82
+ end
83
+
84
+ def update_settings_op(index, settings)
85
+ UpdateIndexSettings.receive(index: index, settings: settings)
86
+ end
87
+
88
+ def delete_index_op(index, obj_type = nil)
89
+ DeleteIndex.receive({ index: index, obj_type: obj_type }.compact_blank)
90
+ end
91
+
92
+ def update_mapping_op(index, obj_type, mapping)
93
+ UpdateIndexMapping.receive(index: index, obj_type: obj_type, mapping: mapping)
94
+ end
95
+
96
+ def alias_index_op (action, index, als)
97
+ AliasIndex.receive(action: action, index: index, alias_name: als)
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,86 @@
1
+ module Wukong
2
+ module Migrate
3
+ class MigrateRunner < Wukong::Runner
4
+ include Wukong::Logging
5
+ include Wukong::Plugin ; log.level = 2
6
+
7
+ description <<-DESC.gsub(/^ {8}/, '').strip
8
+ Use this tool to create and perform database migrations using models
9
+ defined in app/models.
10
+
11
+ Commands:
12
+
13
+ generate <name> Creates a migration file for you under db/migrate/<name>
14
+ perform <name> Runs a specified migration
15
+ all Runs all migrations
16
+ DESC
17
+
18
+ class << self
19
+ def configure(env, prog)
20
+ env.define :debug, type: :boolean, default: false, description: 'Run in debug mode'
21
+ env.define :db, required: true, description: 'The database to apply the migration to'
22
+ env.define :force, type: :boolean, default: false, description: 'Continue migrating through errors'
23
+ end
24
+ end
25
+
26
+ def command
27
+ args.first
28
+ end
29
+
30
+ def specified_migration
31
+ args[1] or die('Must specify a migration when using this command', 1)
32
+ end
33
+
34
+ def migration_file_dir
35
+ Wukong::Deploy.root.join('db/migrate')
36
+ end
37
+
38
+ def database_options
39
+ opts = settings.to_hash
40
+ opts.merge(opts.delete(settings.db.to_sym) || {})
41
+ end
42
+
43
+ def generate_migration_file(name, database)
44
+ m_file = migration_file_dir.join(name + '.rb').to_s
45
+ log.info "Creating migration: #{m_file}"
46
+ case database
47
+ when 'elasticsearch'
48
+ File.open(m_file, 'w'){ |f| f.puts EsMigration.template(name) }
49
+ when 'hbase'
50
+ File.open(m_file, 'w'){ |f| f.puts HbaseMigration.template(name) }
51
+ end
52
+ end
53
+
54
+ def load_all_migration_files!
55
+ migration_file_dir.children.each do |m_file|
56
+ Kernel.load m_file.to_s if m_file.extname == '.rb'
57
+ end
58
+ end
59
+
60
+ def perform_migration(*names, options)
61
+ names.each do |name|
62
+ migration = Wukong::Migration.retrieve(name)
63
+ migration.write_attribute(:log, self.log)
64
+ migration.perform(options)
65
+ end
66
+ end
67
+
68
+ def run
69
+ case command
70
+ when 'generate'
71
+ generate_migration_file(specified_migration, settings.db)
72
+ when 'perform'
73
+ load_all_migration_files!
74
+ perform_migration(specified_migration, database_options)
75
+ when 'all'
76
+ load_all_migration_files!
77
+ perform_migration(*Wukong::Migration.all_migrations, database_options)
78
+ else
79
+ log.error "Please specify a valid command"
80
+ dump_help_and_exit!
81
+ end
82
+ end
83
+
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,5 @@
1
+ module Wukong
2
+ module Migrate
3
+ VERSION = '0.0.1'
4
+ end
5
+ end
@@ -0,0 +1,118 @@
1
+ require 'spec_helper'
2
+
3
+ describe Gorillib::Model do
4
+
5
+ subject{ TestModel = Class.new(Object){ include Gorillib::Model } }
6
+
7
+ after(:each) do
8
+ Object.send(:remove_const, :TestModel)
9
+ end
10
+
11
+ context '#to_mapping' do
12
+ it 'generates string mappings correctly' do
13
+ subject.class_eval do
14
+ field :foo, String, es_options: { analyzer: 'whitespace' }
15
+ end
16
+ subject.to_mapping.should eq({ properties: { foo: { type: 'string', index: 'not_analyzed', analyzer: 'whitespace' } } })
17
+ end
18
+
19
+ it 'generates integer mappings correctly' do
20
+ subject.class_eval do
21
+ field :foo, Integer, es_options: { precision_step: 2 }
22
+ end
23
+ subject.to_mapping.should eq({ properties: { foo: { type: 'integer', precision_step: 2 } } })
24
+ end
25
+
26
+ it 'generates boolean mappings correctly' do
27
+ subject.class_eval do
28
+ field :foo, :boolean, es_options: { store: 'yes' }
29
+ end
30
+ subject.to_mapping.should eq({ properties: { foo: { type: 'boolean', store: 'yes' } } })
31
+ end
32
+
33
+ it 'generates date mappings correctly' do
34
+ subject.class_eval do
35
+ field :foo, Date, es_options: { format: 'basic_date' }
36
+ end
37
+ subject.to_mapping.should eq({ properties: { foo: { type: 'date', format: 'basic_date' } } })
38
+ end
39
+
40
+ it 'generates float mappings correctly' do
41
+ subject.class_eval do
42
+ field :foo, Float, es_options: { ignore_malformed: true }
43
+ end
44
+ subject.to_mapping.should eq({ properties: { foo: { type: 'float', ignore_malformed: true } } })
45
+ end
46
+
47
+ it 'generates array mappings correctly' do
48
+ subject.class_eval do
49
+ field :foo, Array, of: Float, es_options: { precision_step: 6 }
50
+ end
51
+ subject.to_mapping.should eq({ properties: { foo: { type: 'float', precision_step: 6 } } })
52
+ end
53
+
54
+ it 'generates object mappings correctly' do
55
+ subject.class_eval do
56
+ class Bar
57
+ include Gorillib::Model
58
+ field :baz, String
59
+ end
60
+
61
+ field :bar, Bar
62
+ end
63
+ subject.to_mapping.should eq({ properties: { bar: { properties: { baz: { type: 'string', index: 'not_analyzed' } } } } })
64
+ end
65
+
66
+ it 'handles non-standard fields as strings' do
67
+ subject.class_eval do
68
+ class Baz
69
+ def self.receive(param) param ; end
70
+ end
71
+ field :bar, Baz
72
+ end
73
+ subject.to_mapping.should eq({ properties: { bar: { type: 'string', index: 'not_analyzed' } } })
74
+ end
75
+ end
76
+ end
77
+
78
+ describe Gorillib::Builder do
79
+
80
+ subject{ TestBuilder = Class.new(Object){ include Gorillib::Builder } }
81
+
82
+ after(:each) do
83
+ Object.send(:remove_const, :TestBuilder)
84
+ end
85
+
86
+ it 'allows receive overrides of magic fields' do
87
+ subject.class_eval do
88
+ magic :foo, String
89
+
90
+ def receive_foo param
91
+ super(param + 'bar')
92
+ end
93
+ end
94
+ subject.receive(foo: 'foo').foo.should eq('foobar')
95
+ end
96
+
97
+ it 'allows receive overrides of collection fields' do
98
+ subject.class_eval do
99
+ class Bar
100
+ include Gorillib::Builder
101
+ magic :baz, String
102
+ end
103
+
104
+ collection :bars, Bar
105
+
106
+ def receive_bar(attrs, &block)
107
+ b = Bar.receive(attrs, &block)
108
+ b.baz = 'foo' + b.baz
109
+ b
110
+ end
111
+ end
112
+ subject.new do
113
+ bar(:bar) do
114
+ baz 'baz'
115
+ end
116
+ end.bars[:bar].baz.should eq('foobaz')
117
+ end
118
+ end
@@ -0,0 +1 @@
1
+ require 'wukong-migrate'
@@ -0,0 +1,69 @@
1
+ require 'spec_helper'
2
+
3
+ describe EsMigration do
4
+ include EsHttpOperation::Helpers
5
+
6
+ subject{ described_class }
7
+
8
+ class SimpleModel
9
+ include Gorillib::Model
10
+ field :test_field, Integer
11
+ end
12
+
13
+ context '#combined_operations' do
14
+ it 'handles index operations' do
15
+ subject.new do
16
+ create_index(:foo)
17
+ end.combined_operations.should eq([create_index_op('foo', {})])
18
+ end
19
+
20
+ it 'handles index settings' do
21
+ subject.new do
22
+ update_index(:foo) do
23
+ number_of_replicas 6
24
+ end
25
+ end.combined_operations.should eq([update_settings_op('foo', { number_of_replicas: 6 })])
26
+ end
27
+
28
+ it 'handles delete operations first' do
29
+ subject.new do
30
+ create_index(:foo)
31
+ delete_index(:bar)
32
+ end.combined_operations.should eq([delete_index_op('bar'),
33
+ create_index_op('foo', {})])
34
+ end
35
+
36
+ it 'handles alias operations last' do
37
+ m = subject.new do
38
+ create_index(:foo) do
39
+ alias_to [:superfoo, :superbar]
40
+ end
41
+ delete_index(:bar)
42
+ end.combined_operations.should eq([delete_index_op('bar'),
43
+ create_index_op('foo', {}),
44
+ alias_index_op('add', 'foo', 'superfoo'),
45
+ alias_index_op('add', 'foo', 'superbar')])
46
+ end
47
+
48
+ it 'handles mapping operations' do
49
+ subject.new do
50
+ update_index(:foo) do
51
+ create_mapping(:simple_model)
52
+ end
53
+ end.combined_operations.should eq([update_settings_op('foo', {}),
54
+ update_mapping_op('foo', 'simple_model', SimpleModel.to_mapping)])
55
+
56
+ end
57
+
58
+ it 'handles mapping settings' do
59
+ subject.new do
60
+ update_index(:foo) do
61
+ create_mapping(:simple_model) do
62
+ dynamic true
63
+ end
64
+ end
65
+ end.combined_operations.should eq([update_settings_op('foo', {}),
66
+ update_mapping_op('foo', 'simple_model', SimpleModel.to_mapping.merge(dynamic: true))])
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,77 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'EsHttpOperation' do
4
+
5
+ subject{ EsHttpOperation.receive(index: 'foo') }
6
+
7
+ before(:each) do
8
+ subject.configure_with(host: 'localhost', port: 9200)
9
+ end
10
+
11
+ context '#execute' do
12
+ it 'calls its own http method' do
13
+ subject.should_receive(:call_own_http_method).and_return 'a response'
14
+ subject.execute.should eq('a response')
15
+ end
16
+ end
17
+
18
+ context EsHttpOperation::CreateIndex do
19
+ subject{ described_class.receive(index: 'foo', settings: { foo: 'bar' }) }
20
+
21
+ its(:path) { should eq('/foo/') }
22
+ its(:body) { should eq({ settings: { foo: 'bar' }}) }
23
+ its(:verb) { should eq(:put) }
24
+ its(:info) { should eq('Creating index foo') }
25
+ its(:raw_curl_string) do
26
+ should eq("curl -X PUT 'http://localhost:9200/foo/' -d '{\"settings\":{\"foo\":\"bar\"}}'")
27
+ end
28
+ end
29
+
30
+ context EsHttpOperation::DeleteIndex do
31
+ subject{ described_class.receive(index: 'foo') }
32
+
33
+ its(:path) { should eq('/foo/') }
34
+ its(:body) { should eq(nil) }
35
+ its(:verb) { should eq(:delete) }
36
+ its(:info) { should eq('Deleting index foo') }
37
+ its(:raw_curl_string) do
38
+ should eq("curl -X DELETE 'http://localhost:9200/foo/'")
39
+ end
40
+ end
41
+
42
+ context EsHttpOperation::UpdateIndexSettings do
43
+ subject{ described_class.receive(index: 'foo', settings: { foo: 'bar'}) }
44
+
45
+ its(:path) { should eq('/foo/_settings?') }
46
+ its(:body) { should eq({ index: { foo: 'bar' }}) }
47
+ its(:verb) { should eq(:put) }
48
+ its(:info) { should eq('Updating settings for index foo') }
49
+ its(:raw_curl_string) do
50
+ should eq("curl -X PUT 'http://localhost:9200/foo/_settings?' -d '{\"index\":{\"foo\":\"bar\"}}'")
51
+ end
52
+ end
53
+
54
+ context EsHttpOperation::AliasIndex do
55
+ subject{ described_class.receive(index: 'foo', alias_name: 'bar', action: 'add') }
56
+
57
+ its(:path) { should eq('/_aliases?') }
58
+ its(:body) { should eq({ actions: [{ add: { index: 'foo', alias: 'bar' } }]}) }
59
+ its(:verb) { should eq(:post) }
60
+ its(:info) { should eq('Add alias :bar for index foo') }
61
+ its(:raw_curl_string) do
62
+ should eq("curl -X POST 'http://localhost:9200/_aliases?' -d '{\"actions\":[{\"add\":{\"index\":\"foo\",\"alias\":\"bar\"}}]}'")
63
+ end
64
+ end
65
+
66
+ context EsHttpOperation::UpdateIndexMapping do
67
+ subject{ described_class.receive(index: 'foo', obj_type: 'bar', mapping: { dynamic: true }) }
68
+
69
+ its(:path) { should eq('/foo/bar/_mapping?') }
70
+ its(:body) { should eq({ 'bar' => { dynamic: true } }) }
71
+ its(:verb) { should eq(:put) }
72
+ its(:info) { should eq('Updating bar mapping for index foo') }
73
+ its(:raw_curl_string) do
74
+ should eq("curl -X PUT 'http://localhost:9200/foo/bar/_mapping?' -d '{\"bar\":{\"dynamic\":true}}'")
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,27 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/wukong-migrate/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.name = 'wukong-migrate'
6
+ gem.homepage = 'https://github.com/infochimps-labs/wukong-migrate'
7
+ gem.licenses = ['Apache 2.0']
8
+ gem.email = 'coders@infochimps.com'
9
+ gem.authors = ['Travis Dempsey']
10
+ gem.version = Wukong::Migrate::VERSION
11
+
12
+ gem.summary = 'Wukong utility to push database schema changes based upon your defined models'
13
+ gem.description = <<-DESC.gsub(/^ {4}/, '')
14
+ wukong-migrate, inspired by rails, all up in yer deploy pack, pushing yer schema changes
15
+ DESC
16
+
17
+ gem.files = `git ls-files`.split("\n")
18
+ gem.executables = ['wu-migrate']
19
+ gem.test_files = gem.files.grep(/^spec/)
20
+ gem.require_paths = ['lib']
21
+
22
+ gem.add_dependency('wukong-deploy', '>= 0.1.1')
23
+ gem.add_dependency('gorillib', '~> 0.5')
24
+ gem.add_dependency('httparty', '~> 0.11')
25
+ gem.add_dependency('rake', '>= 0.8.7')
26
+ end
27
+
metadata ADDED
@@ -0,0 +1,124 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: wukong-migrate
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Travis Dempsey
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-07-25 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: wukong-deploy
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 0.1.1
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ! '>='
25
+ - !ruby/object:Gem::Version
26
+ version: 0.1.1
27
+ - !ruby/object:Gem::Dependency
28
+ name: gorillib
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '0.5'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '0.5'
41
+ - !ruby/object:Gem::Dependency
42
+ name: httparty
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: '0.11'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ~>
53
+ - !ruby/object:Gem::Version
54
+ version: '0.11'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: 0.8.7
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ! '>='
67
+ - !ruby/object:Gem::Version
68
+ version: 0.8.7
69
+ description: ! 'wukong-migrate, inspired by rails, all up in yer deploy pack, pushing
70
+ yer schema changes
71
+
72
+ '
73
+ email: coders@infochimps.com
74
+ executables:
75
+ - wu-migrate
76
+ extensions: []
77
+ extra_rdoc_files: []
78
+ files:
79
+ - Gemfile
80
+ - Gemfile.lock
81
+ - Rakefile
82
+ - bin/wu-migrate
83
+ - lib/gorillib/model/elasticsearch_ext.rb
84
+ - lib/wukong-migrate.rb
85
+ - lib/wukong-migrate/dsl.rb
86
+ - lib/wukong-migrate/elasticsearch_fields.rb
87
+ - lib/wukong-migrate/elasticsearch_migration.rb
88
+ - lib/wukong-migrate/elasticsearch_operations.rb
89
+ - lib/wukong-migrate/migrate_runner.rb
90
+ - lib/wukong-migrate/version.rb
91
+ - spec/gorillib/elasticsearch_ext_spec.rb
92
+ - spec/spec_helper.rb
93
+ - spec/wukong-migrate/elasticsearch_migration_spec.rb
94
+ - spec/wukong-migrate/elasticsearch_operations_spec.rb
95
+ - wukong-migrate.gemspec
96
+ homepage: https://github.com/infochimps-labs/wukong-migrate
97
+ licenses:
98
+ - Apache 2.0
99
+ metadata: {}
100
+ post_install_message:
101
+ rdoc_options: []
102
+ require_paths:
103
+ - lib
104
+ required_ruby_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ! '>='
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ required_rubygems_version: !ruby/object:Gem::Requirement
110
+ requirements:
111
+ - - ! '>='
112
+ - !ruby/object:Gem::Version
113
+ version: '0'
114
+ requirements: []
115
+ rubyforge_project:
116
+ rubygems_version: 2.0.5
117
+ signing_key:
118
+ specification_version: 4
119
+ summary: Wukong utility to push database schema changes based upon your defined models
120
+ test_files:
121
+ - spec/gorillib/elasticsearch_ext_spec.rb
122
+ - spec/spec_helper.rb
123
+ - spec/wukong-migrate/elasticsearch_migration_spec.rb
124
+ - spec/wukong-migrate/elasticsearch_operations_spec.rb