onemorecloud-websolr-rails 1.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (166) hide show
  1. data/.document +5 -0
  2. data/.gitignore +15 -0
  3. data/LICENSE +19 -0
  4. data/README.rdoc +69 -0
  5. data/Rakefile +112 -0
  6. data/TESTING_THE_PLUGIN +25 -0
  7. data/VERSION +1 -0
  8. data/bin/websolr +8 -0
  9. data/config/solr.yml +15 -0
  10. data/config/solr_environment.rb +22 -0
  11. data/generate_rdoc.sh +13 -0
  12. data/lib/acts_as_solr.rb +59 -0
  13. data/lib/acts_methods.rb +279 -0
  14. data/lib/class_methods.rb +239 -0
  15. data/lib/common_methods.rb +89 -0
  16. data/lib/deprecation.rb +61 -0
  17. data/lib/instance_methods.rb +166 -0
  18. data/lib/lazy_document.rb +18 -0
  19. data/lib/parser_methods.rb +201 -0
  20. data/lib/search_results.rb +68 -0
  21. data/lib/solr.rb +21 -0
  22. data/lib/solr/connection.rb +179 -0
  23. data/lib/solr/document.rb +78 -0
  24. data/lib/solr/exception.rb +13 -0
  25. data/lib/solr/field.rb +39 -0
  26. data/lib/solr/importer.rb +19 -0
  27. data/lib/solr/importer/array_mapper.rb +26 -0
  28. data/lib/solr/importer/delimited_file_source.rb +38 -0
  29. data/lib/solr/importer/hpricot_mapper.rb +27 -0
  30. data/lib/solr/importer/mapper.rb +51 -0
  31. data/lib/solr/importer/solr_source.rb +43 -0
  32. data/lib/solr/importer/xpath_mapper.rb +35 -0
  33. data/lib/solr/indexer.rb +52 -0
  34. data/lib/solr/request.rb +26 -0
  35. data/lib/solr/request/add_document.rb +63 -0
  36. data/lib/solr/request/base.rb +36 -0
  37. data/lib/solr/request/commit.rb +31 -0
  38. data/lib/solr/request/delete.rb +50 -0
  39. data/lib/solr/request/dismax.rb +46 -0
  40. data/lib/solr/request/index_info.rb +22 -0
  41. data/lib/solr/request/modify_document.rb +51 -0
  42. data/lib/solr/request/optimize.rb +21 -0
  43. data/lib/solr/request/ping.rb +36 -0
  44. data/lib/solr/request/select.rb +56 -0
  45. data/lib/solr/request/spellcheck.rb +30 -0
  46. data/lib/solr/request/standard.rb +402 -0
  47. data/lib/solr/request/update.rb +23 -0
  48. data/lib/solr/response.rb +27 -0
  49. data/lib/solr/response/add_document.rb +17 -0
  50. data/lib/solr/response/base.rb +42 -0
  51. data/lib/solr/response/commit.rb +17 -0
  52. data/lib/solr/response/delete.rb +13 -0
  53. data/lib/solr/response/dismax.rb +8 -0
  54. data/lib/solr/response/index_info.rb +26 -0
  55. data/lib/solr/response/modify_document.rb +17 -0
  56. data/lib/solr/response/optimize.rb +14 -0
  57. data/lib/solr/response/ping.rb +28 -0
  58. data/lib/solr/response/ruby.rb +42 -0
  59. data/lib/solr/response/select.rb +17 -0
  60. data/lib/solr/response/spellcheck.rb +20 -0
  61. data/lib/solr/response/standard.rb +60 -0
  62. data/lib/solr/response/xml.rb +42 -0
  63. data/lib/solr/solrtasks.rb +27 -0
  64. data/lib/solr/util.rb +32 -0
  65. data/lib/solr/xml.rb +44 -0
  66. data/lib/solr_fixtures.rb +13 -0
  67. data/lib/tasks/database.rake +18 -0
  68. data/lib/tasks/solr.rake +137 -0
  69. data/lib/tasks/test.rake +7 -0
  70. data/lib/websolr-rails.rb +1 -0
  71. data/lib/websolr.rb +1 -0
  72. data/lib/websolr_controller.rb +132 -0
  73. data/lib/websolr_option_parser.rb +58 -0
  74. data/lib/websolr_rails.rb +1 -0
  75. data/lib/websolr_rails/tasks.rb +4 -0
  76. data/solr/CHANGES.txt +1207 -0
  77. data/solr/LICENSE.txt +712 -0
  78. data/solr/NOTICE.txt +90 -0
  79. data/solr/etc/jetty.xml +205 -0
  80. data/solr/etc/webdefault.xml +379 -0
  81. data/solr/lib/easymock.jar +0 -0
  82. data/solr/lib/jetty-6.1.3.jar +0 -0
  83. data/solr/lib/jetty-util-6.1.3.jar +0 -0
  84. data/solr/lib/jsp-2.1/ant-1.6.5.jar +0 -0
  85. data/solr/lib/jsp-2.1/core-3.1.1.jar +0 -0
  86. data/solr/lib/jsp-2.1/jsp-2.1.jar +0 -0
  87. data/solr/lib/jsp-2.1/jsp-api-2.1.jar +0 -0
  88. data/solr/lib/servlet-api-2.4.jar +0 -0
  89. data/solr/lib/servlet-api-2.5-6.1.3.jar +0 -0
  90. data/solr/lib/xpp3-1.1.3.4.O.jar +0 -0
  91. data/solr/logs/.empty-dir-for-git +0 -0
  92. data/solr/solr/README.txt +52 -0
  93. data/solr/solr/bin/abc +176 -0
  94. data/solr/solr/bin/abo +176 -0
  95. data/solr/solr/bin/backup +108 -0
  96. data/solr/solr/bin/backupcleaner +142 -0
  97. data/solr/solr/bin/commit +128 -0
  98. data/solr/solr/bin/optimize +129 -0
  99. data/solr/solr/bin/readercycle +129 -0
  100. data/solr/solr/bin/rsyncd-disable +77 -0
  101. data/solr/solr/bin/rsyncd-enable +76 -0
  102. data/solr/solr/bin/rsyncd-start +145 -0
  103. data/solr/solr/bin/rsyncd-stop +105 -0
  104. data/solr/solr/bin/scripts-util +83 -0
  105. data/solr/solr/bin/snapcleaner +148 -0
  106. data/solr/solr/bin/snapinstaller +168 -0
  107. data/solr/solr/bin/snappuller +248 -0
  108. data/solr/solr/bin/snappuller-disable +77 -0
  109. data/solr/solr/bin/snappuller-enable +77 -0
  110. data/solr/solr/bin/snapshooter +109 -0
  111. data/solr/solr/conf/admin-extra.html +31 -0
  112. data/solr/solr/conf/protwords.txt +21 -0
  113. data/solr/solr/conf/schema.xml +126 -0
  114. data/solr/solr/conf/scripts.conf +24 -0
  115. data/solr/solr/conf/solrconfig.xml +458 -0
  116. data/solr/solr/conf/stopwords.txt +57 -0
  117. data/solr/solr/conf/synonyms.txt +31 -0
  118. data/solr/solr/conf/xslt/example.xsl +132 -0
  119. data/solr/solr/conf/xslt/example_atom.xsl +63 -0
  120. data/solr/solr/conf/xslt/example_rss.xsl +62 -0
  121. data/solr/start.jar +0 -0
  122. data/solr/tmp/.empty-dir-for-git +0 -0
  123. data/solr/webapps/solr.war +0 -0
  124. data/test/config/solr.yml +2 -0
  125. data/test/db/connections/mysql/connection.rb +10 -0
  126. data/test/db/connections/sqlite/connection.rb +8 -0
  127. data/test/db/migrate/001_create_books.rb +15 -0
  128. data/test/db/migrate/002_create_movies.rb +12 -0
  129. data/test/db/migrate/003_create_categories.rb +11 -0
  130. data/test/db/migrate/004_create_electronics.rb +16 -0
  131. data/test/db/migrate/005_create_authors.rb +12 -0
  132. data/test/db/migrate/006_create_postings.rb +9 -0
  133. data/test/db/migrate/007_create_posts.rb +13 -0
  134. data/test/db/migrate/008_create_gadgets.rb +11 -0
  135. data/test/fixtures/authors.yml +9 -0
  136. data/test/fixtures/books.yml +13 -0
  137. data/test/fixtures/categories.yml +7 -0
  138. data/test/fixtures/db_definitions/mysql.sql +41 -0
  139. data/test/fixtures/electronics.yml +49 -0
  140. data/test/fixtures/movies.yml +9 -0
  141. data/test/fixtures/postings.yml +10 -0
  142. data/test/functional/acts_as_solr_test.rb +413 -0
  143. data/test/functional/association_indexing_test.rb +37 -0
  144. data/test/functional/faceted_search_test.rb +163 -0
  145. data/test/functional/multi_solr_search_test.rb +51 -0
  146. data/test/models/author.rb +10 -0
  147. data/test/models/book.rb +10 -0
  148. data/test/models/category.rb +8 -0
  149. data/test/models/electronic.rb +21 -0
  150. data/test/models/gadget.rb +9 -0
  151. data/test/models/movie.rb +17 -0
  152. data/test/models/novel.rb +2 -0
  153. data/test/models/post.rb +3 -0
  154. data/test/models/posting.rb +11 -0
  155. data/test/test_helper.rb +51 -0
  156. data/test/unit/acts_methods_shoulda.rb +70 -0
  157. data/test/unit/class_methods_shoulda.rb +88 -0
  158. data/test/unit/common_methods_shoulda.rb +112 -0
  159. data/test/unit/instance_methods_shoulda.rb +326 -0
  160. data/test/unit/lazy_document_shoulda.rb +35 -0
  161. data/test/unit/parser_instance.rb +19 -0
  162. data/test/unit/parser_methods_shoulda.rb +278 -0
  163. data/test/unit/solr_instance.rb +46 -0
  164. data/test/unit/test_helper.rb +14 -0
  165. data/websolr-rails.gemspec +241 -0
  166. metadata +263 -0
@@ -0,0 +1,148 @@
1
+ #!/bin/bash
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one or more
4
+ # contributor license agreements. See the NOTICE file distributed with
5
+ # this work for additional information regarding copyright ownership.
6
+ # The ASF licenses this file to You under the Apache License, Version 2.0
7
+ # (the "License"); you may not use this file except in compliance with
8
+ # the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ # Shell script to clean up snapshots of a Solr Lucene collection.
19
+
20
+ orig_dir=$(pwd)
21
+ cd ${0%/*}/..
22
+ solr_root=$(pwd)
23
+ cd ${orig_dir}
24
+
25
+ unset days num data_dir user verbose debug
26
+ . ${solr_root}/bin/scripts-util
27
+
28
+ # set up variables
29
+ prog=${0##*/}
30
+ log=${solr_root}/logs/${prog}.log
31
+
32
+ # define usage string
33
+ USAGE="\
34
+ usage: $prog -D <days> | -N <num> [-d dir] [-u username] [-v]
35
+ -D <days> cleanup snapshots more than <days> days old
36
+ -N <num> keep the most recent <num> number of snapshots and
37
+ cleanup up the remaining ones that are not being pulled
38
+ -d specify directory holding index data
39
+ -u specify user to sudo to before running script
40
+ -v increase verbosity
41
+ -V output debugging info
42
+ "
43
+
44
+ # parse args
45
+ while getopts D:N:d:u:vV OPTION
46
+ do
47
+ case $OPTION in
48
+ D)
49
+ days="$OPTARG"
50
+ ;;
51
+ N)
52
+ num="$OPTARG"
53
+ ;;
54
+ d)
55
+ data_dir="$OPTARG"
56
+ ;;
57
+ u)
58
+ user="$OPTARG"
59
+ ;;
60
+ v)
61
+ verbose="v"
62
+ ;;
63
+ V)
64
+ debug="V"
65
+ ;;
66
+ *)
67
+ echo "$USAGE"
68
+ exit 1
69
+ esac
70
+ done
71
+
72
+ [[ -n $debug ]] && set -x
73
+
74
+ if [[ -z ${days} && -z ${num} ]]
75
+ then
76
+ echo "$USAGE"
77
+ exit 1
78
+ fi
79
+
80
+ fixUser "$@"
81
+
82
+ # use default value for data_dir if not specified
83
+ # relative path starts at ${solr_root}
84
+ if [[ -z ${data_dir} ]]
85
+ then
86
+ data_dir=${solr_root}/data
87
+ elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]]
88
+ then
89
+ data_dir=${solr_root}/${data_dir}
90
+ fi
91
+
92
+ function remove
93
+ {
94
+ syncing=`ps -fwwwu ${user}|grep -w rsync|grep -v grep|grep -w $1`
95
+ if [[ -n $syncing ]]
96
+ then
97
+ logMessage $1 not removed - rsync in progress
98
+ else
99
+ logMessage removing snapshot $1
100
+ /bin/rm -rf $1
101
+ fi
102
+ }
103
+
104
+ start=`date +"%s"`
105
+
106
+ logMessage started by $oldwhoami
107
+ logMessage command: $0 $@
108
+
109
+ # trap control-c
110
+ trap 'echo "caught INT/TERM, exiting now but partial cleanup may have already occured";logExit aborted 13' INT TERM
111
+
112
+ if [[ -n ${days} ]]
113
+ then
114
+ #is maxdepth supported?
115
+ find ${data_dir} -maxdepth 0 -name foobar >/dev/null 2>&1
116
+ if [ $? = 0 ]; then
117
+ maxdepth="-maxdepth 1"
118
+ else
119
+ unset maxdepth
120
+ fi
121
+
122
+ logMessage cleaning up snapshots more than ${days} days old
123
+ for i in `find ${data_dir} ${maxdepth} -name 'snapshot.*' -mtime +${days} -print`
124
+ do
125
+ remove $i
126
+ done
127
+ elif [[ -n ${num} ]]
128
+ then
129
+ logMessage cleaning up all snapshots except for the most recent ${num} ones
130
+ unset snapshots count
131
+ snapshots=`ls -cd ${data_dir}/snapshot.* 2>/dev/null`
132
+ if [[ $? == 0 ]]
133
+ then
134
+ count=`echo $snapshots|wc -w`
135
+ startpos=`expr $num + 1`
136
+ if [[ $count -gt $num ]]
137
+ then
138
+ for i in `echo $snapshots|cut -f${startpos}- -d" "`
139
+ do
140
+ remove $i
141
+ done
142
+ fi
143
+ fi
144
+ fi
145
+
146
+ logExit ended 0
147
+
148
+
@@ -0,0 +1,168 @@
1
+ #!/bin/bash
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one or more
4
+ # contributor license agreements. See the NOTICE file distributed with
5
+ # this work for additional information regarding copyright ownership.
6
+ # The ASF licenses this file to You under the Apache License, Version 2.0
7
+ # (the "License"); you may not use this file except in compliance with
8
+ # the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ # Shell script to install a snapshot into place as the Lucene collection
19
+ # for a Solr server
20
+
21
+ orig_dir=$(pwd)
22
+ cd ${0%/*}/..
23
+ solr_root=$(pwd)
24
+ cd ${orig_dir}
25
+
26
+ unset master_host master_status_dir data_dir user verbose debug
27
+ . ${solr_root}/bin/scripts-util
28
+
29
+ # set up variables
30
+ prog=${0##*/}
31
+ log=${solr_root}/logs/${prog}.log
32
+
33
+ # define usage string
34
+ USAGE="\
35
+ usage: $prog [-M master] [-S sdir] [-d dir] [-u username] [-v]
36
+ -M master specify hostname of master server from where to pull index
37
+ snapshot
38
+ -S specify directory holding snapshot status on master server
39
+ -d specify directory holding index data on local machine
40
+ -u specify user to sudo to before running script
41
+ -v increase verbosity
42
+ -V output debugging info
43
+ "
44
+
45
+ # parse args
46
+ while getopts M:S:d:u:vV OPTION
47
+ do
48
+ case $OPTION in
49
+ M)
50
+ master_host="$OPTARG"
51
+ ;;
52
+ S)
53
+ master_status_dir="$OPTARG"
54
+ ;;
55
+ d)
56
+ data_dir="$OPTARG"
57
+ ;;
58
+ u)
59
+ user="$OPTARG"
60
+ ;;
61
+ v)
62
+ verbose="v"
63
+ ;;
64
+ V)
65
+ debug="V"
66
+ ;;
67
+ *)
68
+ echo "$USAGE"
69
+ exit 1
70
+ esac
71
+ done
72
+
73
+ [[ -n $debug ]] && set -x
74
+
75
+ if [[ -z ${master_host} ]]
76
+ then
77
+ echo "name of master server missing in $confFile or command line."
78
+ echo "$USAGE"
79
+ exit 1
80
+ fi
81
+
82
+ if [[ -z ${master_status_dir} ]]
83
+ then
84
+ echo "directory holding snapshot status on master server missing in $confFile or command line."
85
+ echo "$USAGE"
86
+ exit 1
87
+ fi
88
+
89
+ fixUser "$@"
90
+
91
+ # use default value for data_dir if not specified
92
+ # relative path starts at ${solr_root}
93
+ if [[ -z ${data_dir} ]]
94
+ then
95
+ data_dir=${solr_root}/data
96
+ elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]]
97
+ then
98
+ data_dir=${solr_root}/${data_dir}
99
+ fi
100
+
101
+ # assume relative path to start at ${solr_root}
102
+ if [[ "`echo ${master_status_dir}|cut -c1`" != "/" ]]
103
+ then
104
+ master_status_dir=${solr_root}/${master_status_dir}
105
+ fi
106
+
107
+ start=`date +"%s"`
108
+
109
+ logMessage started by $oldwhoami
110
+ logMessage command: $0 $@
111
+
112
+ # get directory name of latest snapshot
113
+ name=`ls ${data_dir}|grep 'snapshot\.'|grep -v wip|sort -r|head -1`
114
+
115
+ # clean up after INT/TERM
116
+ trap 'echo "caught INT/TERM, exiting now but partial installation may have already occured";/bin/rm -rf ${data_dir"/index.tmp$$;logExit aborted 13' INT TERM
117
+
118
+ # is there a snapshot
119
+ if [[ "${name}" == "" ]]
120
+ then
121
+ logMessage no shapshot available
122
+ logExit ended 0
123
+ fi
124
+
125
+ name=${data_dir}/${name}
126
+
127
+ # has snapshot already been installed
128
+ if [[ ${name} == `cat ${solr_root}/logs/snapshot.current 2>/dev/null` ]]
129
+ then
130
+ logMessage latest snapshot ${name} already installed
131
+ logExit ended 0
132
+ fi
133
+
134
+ # make sure master has directory for hold slaves stats/state
135
+ if
136
+ ! ssh -o StrictHostKeyChecking=no ${master_host} mkdir -p ${master_status_dir}
137
+ then
138
+ logMessage failed to ssh to master ${master_host}, snapshot status not updated on master
139
+ fi
140
+
141
+ # install using hard links into temporary directory
142
+ # remove original index and then atomically copy new one into place
143
+ logMessage installing snapshot ${name}
144
+ cp -lr ${name}/ ${data_dir}/index.tmp$$
145
+ /bin/rm -rf ${data_dir}/index
146
+ mv -f ${data_dir}/index.tmp$$ ${data_dir}/index
147
+
148
+ # update distribution stats
149
+ echo ${name} > ${solr_root}/logs/snapshot.current
150
+
151
+ # push stats/state to master
152
+ if
153
+ ! scp -q -o StrictHostKeyChecking=no ${solr_root}/logs/snapshot.current ${master_host}:${master_status_dir}/snapshot.current.`uname -n`
154
+ then
155
+ logMessage failed to ssh to master ${master_host}, snapshot status not updated on master
156
+ fi
157
+
158
+ # notify Solr to open a new Searcher
159
+ logMessage notifing Solr to open a new Searcher
160
+ ${solr_root}/bin/commit
161
+ if [[ $? != 0 ]]
162
+ then
163
+ logMessage failed to connect to Solr server
164
+ logMessage snapshot installed but Solr server has not open a new Searcher
165
+ logExit failed 1
166
+ fi
167
+
168
+ logExit ended 0
@@ -0,0 +1,248 @@
1
+ #!/bin/bash
2
+ #
3
+ # Licensed to the Apache Software Foundation (ASF) under one or more
4
+ # contributor license agreements. See the NOTICE file distributed with
5
+ # this work for additional information regarding copyright ownership.
6
+ # The ASF licenses this file to You under the Apache License, Version 2.0
7
+ # (the "License"); you may not use this file except in compliance with
8
+ # the License. You may obtain a copy of the License at
9
+ #
10
+ # http://www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS,
14
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+ # See the License for the specific language governing permissions and
16
+ # limitations under the License.
17
+ #
18
+ # Shell script to copy snapshots of a Solr Lucene collection from the master
19
+
20
+ orig_dir=$(pwd)
21
+ cd ${0%/*}/..
22
+ solr_root=$(pwd)
23
+ cd ${orig_dir}
24
+
25
+ unset master_host rsyncd_port master_data_dir master_status_dir snap_name
26
+ unset sizeonly stats data_dir user verbose debug compress startStatus
27
+ . ${solr_root}/bin/scripts-util
28
+
29
+ # set up variables
30
+ prog=${0##*/}
31
+ log=${solr_root}/logs/${prog}.log
32
+
33
+ # define usage string
34
+ USAGE="\
35
+ usage: $prog [-M master] [-P portnum] [-D mdir] [-S sdir] [-n snapshot] [-d dir] [-u username] [-svz]
36
+ -M master specify hostname of master server from where to pull index
37
+ snapshot
38
+ -P port specify rsyncd port number of master server from where to
39
+ pull index snapshot
40
+ -D specify directory holding index data on master server
41
+ -S specify directory holding snapshot status on master server
42
+ -n snapshot pull a specific snapshot by name
43
+ -d specify directory holding index data on local machine
44
+ -u specify user to sudo to before running script
45
+ -s use the --size-only option with rsync
46
+ -v increase verbosity (-vv show file transfer stats also)
47
+ -V output debugging info
48
+ -z enable compression of data
49
+ "
50
+
51
+ # parse args
52
+ while getopts M:P:D:S:n:d:u:svVz OPTION
53
+ do
54
+ case $OPTION in
55
+ M)
56
+ master_host="$OPTARG"
57
+ ;;
58
+ P)
59
+ rsyncd_port="$OPTARG"
60
+ ;;
61
+ D)
62
+ master_data_dir="$OPTARG"
63
+ ;;
64
+ S)
65
+ master_status_dir="$OPTARG"
66
+ ;;
67
+ n)
68
+ snap_name="$OPTARG"
69
+ ;;
70
+ d)
71
+ data_dir="$OPTARG"
72
+ ;;
73
+ u)
74
+ user="$OPTARG"
75
+ ;;
76
+ s)
77
+ sizeonly="--size-only"
78
+ ;;
79
+ v)
80
+ [[ -n $verbose ]] && stats="--stats" || verbose=v
81
+ ;;
82
+ V)
83
+ debug="V"
84
+ ;;
85
+ z)
86
+ compress="z"
87
+ ;;
88
+ *)
89
+ echo "$USAGE"
90
+ exit 1
91
+ esac
92
+ done
93
+
94
+ [[ -n $debug ]] && set -x
95
+
96
+ if [[ -z ${master_host} ]]
97
+ then
98
+ echo "name of master server missing in $confFile or command line."
99
+ echo "$USAGE"
100
+ exit 1
101
+ fi
102
+
103
+ # try to determine rsyncd port number from $confFile if not specified on
104
+ # command line, default to solr_port+10000
105
+ if [[ -z ${rsyncd_port} ]]
106
+ then
107
+ if [[ "${solr_port}" ]]
108
+ then
109
+ rsyncd_port=`expr 10000 + ${solr_port}`
110
+ else
111
+ echo "rsyncd port number of master server missing in $confFile or command line."
112
+ echo "$USAGE"
113
+ exit 1
114
+ fi
115
+ fi
116
+
117
+ if [[ -z ${master_data_dir} ]]
118
+ then
119
+ echo "directory holding index data on master server missing in $confFile or command line."
120
+ echo "$USAGE"
121
+ exit 1
122
+ fi
123
+
124
+ if [[ -z ${master_status_dir} ]]
125
+ then
126
+ echo "directory holding snapshot status on master server missing in $confFile or command line."
127
+ echo "$USAGE"
128
+ exit 1
129
+ fi
130
+
131
+ fixUser "$@"
132
+
133
+ # use default value for data_dir if not specified
134
+ # relative path starts at ${solr_root}
135
+ if [[ -z ${data_dir} ]]
136
+ then
137
+ data_dir=${solr_root}/data
138
+ elif [[ "`echo ${data_dir}|cut -c1`" != "/" ]]
139
+ then
140
+ data_dir=${solr_root}/${data_dir}
141
+ fi
142
+
143
+ # assume relative path to start at ${solr_root}
144
+ if [[ "`echo ${master_data_dir}|cut -c1`" != "/" ]]
145
+ then
146
+ master_data_dir=${solr_root}/${master_data_dir}
147
+ fi
148
+ if [[ "`echo ${master_status_dir}|cut -c1`" != "/" ]]
149
+ then
150
+ master_status_dir=${solr_root}/${master_status_dir}
151
+ fi
152
+
153
+ # push stats/state to master if necessary
154
+ function pushStatus
155
+ {
156
+ scp -q -o StrictHostKeyChecking=no ${solr_root}/logs/snappuller.status ${master_host}:${master_status_dir}/snapshot.status.`uname -n`
157
+ }
158
+
159
+ start=`date +"%s"`
160
+
161
+ logMessage started by $oldwhoami
162
+ logMessage command: $0 $@
163
+
164
+ if [[ ! -f ${solr_root}/logs/snappuller-enabled ]]
165
+ then
166
+ logMessage snappuller disabled
167
+ exit 1
168
+ fi
169
+
170
+ # make sure we can ssh to master
171
+ if
172
+ ! ssh -o StrictHostKeyChecking=no ${master_host} id 1>/dev/null 2>&1
173
+ then
174
+ logMessage failed to ssh to master ${master_host}
175
+ exit 1
176
+ fi
177
+
178
+ # get directory name of latest snapshot if not specified on command line
179
+ if [[ -z ${snap_name} ]]
180
+ then
181
+ snap_name=`ssh -o StrictHostKeyChecking=no ${master_host} "ls ${master_data_dir}|grep 'snapshot\.'|grep -v wip|sort -r|head -1"`
182
+ fi
183
+ if [[ "${snap_name}" == "" ]]
184
+ then
185
+ logMessage no snapshot available on ${master_host} in ${master_data_dir}
186
+ logExit ended 0
187
+ else
188
+ name=`basename ${snap_name}`
189
+ fi
190
+
191
+ # clean up after INT/TERM
192
+ trap 'echo cleaning up, please wait ...;/bin/rm -rf ${data_dir}/${name} ${data_dir}/${name}-wip;echo ${startStatus} aborted:$(timeStamp)>${solr_root}/logs/snappuller.status;pushStatus;logExit aborted 13' INT TERM
193
+
194
+ if [[ -d ${data_dir}/${name} || -d ${data_dir}/${name}-wip ]]
195
+ then
196
+ logMessage no new snapshot available on ${master_host} in ${master_data_dir}
197
+ logExit ended 0
198
+ fi
199
+
200
+ # take a snapshot of current index so that only modified files will be rsync-ed
201
+ # put the snapshot in the 'work-in-progress" directory to prevent it from
202
+ # being installed while the copying is still in progress
203
+ cp -lr ${data_dir}/index ${data_dir}/${name}-wip
204
+ # force rsync of segments and .del files since we are doing size-only
205
+ if [[ -n ${sizeonly} ]]
206
+ then
207
+ rm -f ${data_dir}/${name}-wip/segments
208
+ rm -f ${data_dir}/${name}-wip/*.del
209
+ fi
210
+
211
+ logMessage pulling snapshot ${name}
212
+
213
+ # make sure master has directory for hold slaves stats/state
214
+ ssh -o StrictHostKeyChecking=no ${master_host} mkdir -p ${master_status_dir}
215
+
216
+ # start new distribution stats
217
+ rsyncStart=`date`
218
+ startTimestamp=`date -d "$rsyncStart" +'%Y%m%d-%H%M%S'`
219
+ rsyncStartSec=`date -d "$rsyncStart" +'%s'`
220
+ startStatus="rsync of `basename ${name}` started:$startTimestamp"
221
+ echo ${startStatus} > ${solr_root}/logs/snappuller.status
222
+ pushStatus
223
+
224
+ # rsync over files that have changed
225
+ rsync -Wa${verbose}${compress} --delete ${sizeonly} \
226
+ ${stats} rsync://${master_host}:${rsyncd_port}/solr/${name}/ ${data_dir}/${name}-wip
227
+
228
+ rc=$?
229
+ rsyncEnd=`date`
230
+ endTimestamp=`date -d "$rsyncEnd" +'%Y%m%d-%H%M%S'`
231
+ rsyncEndSec=`date -d "$rsyncEnd" +'%s'`
232
+ elapsed=`expr $rsyncEndSec - $rsyncStartSec`
233
+ if [[ $rc != 0 ]]
234
+ then
235
+ logMessage rsync failed
236
+ /bin/rm -rf ${data_dir}/${name}-wip
237
+ echo ${startStatus} failed:$endTimestamp > ${solr_root}/logs/snappuller.status
238
+ pushStatus
239
+ logExit failed 1
240
+ fi
241
+
242
+ # move into place atomically
243
+ mv ${data_dir}/${name}-wip ${data_dir}/${name}
244
+
245
+ # finish new distribution stats`
246
+ echo ${startStatus} ended:$endTimestamp rsync-elapsed:${elapsed} > ${solr_root}/logs/snappuller.status
247
+ pushStatus
248
+ logExit ended 0