logstash-lib 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (419) hide show
  1. data/.gitignore +24 -0
  2. data/.tailor +8 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG +1185 -0
  5. data/CONTRIBUTING.md +61 -0
  6. data/CONTRIBUTORS +79 -0
  7. data/LICENSE +14 -0
  8. data/Makefile +460 -0
  9. data/README.md +120 -0
  10. data/STYLE.md +96 -0
  11. data/bin/logstash +37 -0
  12. data/bin/logstash-test +4 -0
  13. data/bin/logstash-web +4 -0
  14. data/bin/logstash.lib.sh +78 -0
  15. data/bot/check_pull_changelog.rb +89 -0
  16. data/docs/configuration.md +260 -0
  17. data/docs/docgen.rb +242 -0
  18. data/docs/extending/example-add-a-new-filter.md +121 -0
  19. data/docs/extending/index.md +91 -0
  20. data/docs/flags.md +43 -0
  21. data/docs/generate_index.rb +28 -0
  22. data/docs/index.html.erb +56 -0
  23. data/docs/learn.md +46 -0
  24. data/docs/life-of-an-event.md +109 -0
  25. data/docs/logging-tool-comparisons.md +60 -0
  26. data/docs/plugin-doc.html.erb +91 -0
  27. data/docs/plugin-milestones.md +41 -0
  28. data/docs/plugin-synopsis.html.erb +24 -0
  29. data/docs/release-engineering.md +46 -0
  30. data/docs/release-test-results.md +14 -0
  31. data/docs/repositories.md +35 -0
  32. data/docs/tutorials/10-minute-walkthrough/apache-elasticsearch.conf +35 -0
  33. data/docs/tutorials/10-minute-walkthrough/apache-parse.conf +33 -0
  34. data/docs/tutorials/10-minute-walkthrough/apache_log.1 +1 -0
  35. data/docs/tutorials/10-minute-walkthrough/apache_log.2.bz2 +0 -0
  36. data/docs/tutorials/10-minute-walkthrough/hello-search.conf +25 -0
  37. data/docs/tutorials/10-minute-walkthrough/hello.conf +16 -0
  38. data/docs/tutorials/10-minute-walkthrough/index.md +124 -0
  39. data/docs/tutorials/10-minute-walkthrough/step-5-output.txt +17 -0
  40. data/docs/tutorials/getting-started-centralized-overview-diagram.png +0 -0
  41. data/docs/tutorials/getting-started-centralized-overview-diagram.xml +1 -0
  42. data/docs/tutorials/getting-started-centralized.md +217 -0
  43. data/docs/tutorials/getting-started-simple.md +200 -0
  44. data/docs/tutorials/just-enough-rabbitmq-for-logstash.md +201 -0
  45. data/docs/tutorials/media/frontend-response-codes.png +0 -0
  46. data/docs/tutorials/metrics-from-logs.md +84 -0
  47. data/docs/tutorials/zeromq.md +118 -0
  48. data/extract_services.rb +29 -0
  49. data/gembag.rb +64 -0
  50. data/lib/logstash-event.rb +2 -0
  51. data/lib/logstash.rb +4 -0
  52. data/lib/logstash/JRUBY-6970-openssl.rb +22 -0
  53. data/lib/logstash/JRUBY-6970.rb +102 -0
  54. data/lib/logstash/agent.rb +305 -0
  55. data/lib/logstash/certs/cacert.pem +3895 -0
  56. data/lib/logstash/codecs/base.rb +49 -0
  57. data/lib/logstash/codecs/compress_spooler.rb +50 -0
  58. data/lib/logstash/codecs/dots.rb +18 -0
  59. data/lib/logstash/codecs/edn.rb +28 -0
  60. data/lib/logstash/codecs/edn_lines.rb +36 -0
  61. data/lib/logstash/codecs/fluent.rb +55 -0
  62. data/lib/logstash/codecs/graphite.rb +114 -0
  63. data/lib/logstash/codecs/json.rb +41 -0
  64. data/lib/logstash/codecs/json_lines.rb +52 -0
  65. data/lib/logstash/codecs/json_spooler.rb +22 -0
  66. data/lib/logstash/codecs/line.rb +58 -0
  67. data/lib/logstash/codecs/msgpack.rb +43 -0
  68. data/lib/logstash/codecs/multiline.rb +189 -0
  69. data/lib/logstash/codecs/netflow.rb +342 -0
  70. data/lib/logstash/codecs/netflow/util.rb +212 -0
  71. data/lib/logstash/codecs/noop.rb +19 -0
  72. data/lib/logstash/codecs/oldlogstashjson.rb +56 -0
  73. data/lib/logstash/codecs/plain.rb +48 -0
  74. data/lib/logstash/codecs/rubydebug.rb +22 -0
  75. data/lib/logstash/codecs/spool.rb +38 -0
  76. data/lib/logstash/config/Makefile +4 -0
  77. data/lib/logstash/config/config_ast.rb +380 -0
  78. data/lib/logstash/config/file.rb +39 -0
  79. data/lib/logstash/config/grammar.rb +3504 -0
  80. data/lib/logstash/config/grammar.treetop +241 -0
  81. data/lib/logstash/config/mixin.rb +464 -0
  82. data/lib/logstash/config/registry.rb +13 -0
  83. data/lib/logstash/config/test.conf +18 -0
  84. data/lib/logstash/errors.rb +10 -0
  85. data/lib/logstash/event.rb +262 -0
  86. data/lib/logstash/filters/advisor.rb +178 -0
  87. data/lib/logstash/filters/alter.rb +173 -0
  88. data/lib/logstash/filters/anonymize.rb +93 -0
  89. data/lib/logstash/filters/base.rb +190 -0
  90. data/lib/logstash/filters/checksum.rb +50 -0
  91. data/lib/logstash/filters/cidr.rb +76 -0
  92. data/lib/logstash/filters/cipher.rb +145 -0
  93. data/lib/logstash/filters/clone.rb +35 -0
  94. data/lib/logstash/filters/collate.rb +114 -0
  95. data/lib/logstash/filters/csv.rb +94 -0
  96. data/lib/logstash/filters/date.rb +244 -0
  97. data/lib/logstash/filters/dns.rb +201 -0
  98. data/lib/logstash/filters/drop.rb +32 -0
  99. data/lib/logstash/filters/elapsed.rb +256 -0
  100. data/lib/logstash/filters/elasticsearch.rb +73 -0
  101. data/lib/logstash/filters/environment.rb +27 -0
  102. data/lib/logstash/filters/extractnumbers.rb +84 -0
  103. data/lib/logstash/filters/gelfify.rb +52 -0
  104. data/lib/logstash/filters/geoip.rb +145 -0
  105. data/lib/logstash/filters/grep.rb +153 -0
  106. data/lib/logstash/filters/grok.rb +425 -0
  107. data/lib/logstash/filters/grokdiscovery.rb +75 -0
  108. data/lib/logstash/filters/i18n.rb +51 -0
  109. data/lib/logstash/filters/json.rb +90 -0
  110. data/lib/logstash/filters/json_encode.rb +52 -0
  111. data/lib/logstash/filters/kv.rb +232 -0
  112. data/lib/logstash/filters/metaevent.rb +68 -0
  113. data/lib/logstash/filters/metrics.rb +237 -0
  114. data/lib/logstash/filters/multiline.rb +241 -0
  115. data/lib/logstash/filters/mutate.rb +399 -0
  116. data/lib/logstash/filters/noop.rb +21 -0
  117. data/lib/logstash/filters/prune.rb +149 -0
  118. data/lib/logstash/filters/punct.rb +32 -0
  119. data/lib/logstash/filters/railsparallelrequest.rb +86 -0
  120. data/lib/logstash/filters/range.rb +142 -0
  121. data/lib/logstash/filters/ruby.rb +42 -0
  122. data/lib/logstash/filters/sleep.rb +111 -0
  123. data/lib/logstash/filters/split.rb +64 -0
  124. data/lib/logstash/filters/sumnumbers.rb +73 -0
  125. data/lib/logstash/filters/syslog_pri.rb +107 -0
  126. data/lib/logstash/filters/translate.rb +121 -0
  127. data/lib/logstash/filters/unique.rb +29 -0
  128. data/lib/logstash/filters/urldecode.rb +57 -0
  129. data/lib/logstash/filters/useragent.rb +112 -0
  130. data/lib/logstash/filters/uuid.rb +58 -0
  131. data/lib/logstash/filters/xml.rb +139 -0
  132. data/lib/logstash/filters/zeromq.rb +123 -0
  133. data/lib/logstash/filterworker.rb +122 -0
  134. data/lib/logstash/inputs/base.rb +125 -0
  135. data/lib/logstash/inputs/collectd.rb +306 -0
  136. data/lib/logstash/inputs/drupal_dblog.rb +323 -0
  137. data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb +66 -0
  138. data/lib/logstash/inputs/elasticsearch.rb +140 -0
  139. data/lib/logstash/inputs/eventlog.rb +129 -0
  140. data/lib/logstash/inputs/eventlog/racob_fix.rb +44 -0
  141. data/lib/logstash/inputs/exec.rb +69 -0
  142. data/lib/logstash/inputs/file.rb +146 -0
  143. data/lib/logstash/inputs/ganglia.rb +127 -0
  144. data/lib/logstash/inputs/ganglia/gmondpacket.rb +146 -0
  145. data/lib/logstash/inputs/ganglia/xdr.rb +327 -0
  146. data/lib/logstash/inputs/gelf.rb +138 -0
  147. data/lib/logstash/inputs/gemfire.rb +222 -0
  148. data/lib/logstash/inputs/generator.rb +97 -0
  149. data/lib/logstash/inputs/graphite.rb +41 -0
  150. data/lib/logstash/inputs/heroku.rb +51 -0
  151. data/lib/logstash/inputs/imap.rb +136 -0
  152. data/lib/logstash/inputs/irc.rb +84 -0
  153. data/lib/logstash/inputs/log4j.rb +136 -0
  154. data/lib/logstash/inputs/lumberjack.rb +53 -0
  155. data/lib/logstash/inputs/pipe.rb +57 -0
  156. data/lib/logstash/inputs/rabbitmq.rb +126 -0
  157. data/lib/logstash/inputs/rabbitmq/bunny.rb +118 -0
  158. data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb +1 -0
  159. data/lib/logstash/inputs/rabbitmq/march_hare.rb +129 -0
  160. data/lib/logstash/inputs/redis.rb +263 -0
  161. data/lib/logstash/inputs/relp.rb +106 -0
  162. data/lib/logstash/inputs/s3.rb +279 -0
  163. data/lib/logstash/inputs/snmptrap.rb +87 -0
  164. data/lib/logstash/inputs/sqlite.rb +185 -0
  165. data/lib/logstash/inputs/sqs.rb +172 -0
  166. data/lib/logstash/inputs/stdin.rb +46 -0
  167. data/lib/logstash/inputs/stomp.rb +84 -0
  168. data/lib/logstash/inputs/syslog.rb +237 -0
  169. data/lib/logstash/inputs/tcp.rb +231 -0
  170. data/lib/logstash/inputs/threadable.rb +18 -0
  171. data/lib/logstash/inputs/twitter.rb +82 -0
  172. data/lib/logstash/inputs/udp.rb +81 -0
  173. data/lib/logstash/inputs/unix.rb +163 -0
  174. data/lib/logstash/inputs/varnishlog.rb +48 -0
  175. data/lib/logstash/inputs/websocket.rb +50 -0
  176. data/lib/logstash/inputs/wmi.rb +72 -0
  177. data/lib/logstash/inputs/xmpp.rb +81 -0
  178. data/lib/logstash/inputs/zenoss.rb +143 -0
  179. data/lib/logstash/inputs/zeromq.rb +165 -0
  180. data/lib/logstash/kibana.rb +113 -0
  181. data/lib/logstash/loadlibs.rb +9 -0
  182. data/lib/logstash/logging.rb +89 -0
  183. data/lib/logstash/monkeypatches-for-bugs.rb +2 -0
  184. data/lib/logstash/monkeypatches-for-debugging.rb +47 -0
  185. data/lib/logstash/monkeypatches-for-performance.rb +66 -0
  186. data/lib/logstash/multiqueue.rb +53 -0
  187. data/lib/logstash/namespace.rb +16 -0
  188. data/lib/logstash/outputs/base.rb +120 -0
  189. data/lib/logstash/outputs/boundary.rb +116 -0
  190. data/lib/logstash/outputs/circonus.rb +78 -0
  191. data/lib/logstash/outputs/cloudwatch.rb +351 -0
  192. data/lib/logstash/outputs/csv.rb +55 -0
  193. data/lib/logstash/outputs/datadog.rb +93 -0
  194. data/lib/logstash/outputs/datadog_metrics.rb +123 -0
  195. data/lib/logstash/outputs/elasticsearch.rb +332 -0
  196. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +44 -0
  197. data/lib/logstash/outputs/elasticsearch_http.rb +256 -0
  198. data/lib/logstash/outputs/elasticsearch_river.rb +214 -0
  199. data/lib/logstash/outputs/email.rb +299 -0
  200. data/lib/logstash/outputs/exec.rb +40 -0
  201. data/lib/logstash/outputs/file.rb +180 -0
  202. data/lib/logstash/outputs/ganglia.rb +75 -0
  203. data/lib/logstash/outputs/gelf.rb +208 -0
  204. data/lib/logstash/outputs/gemfire.rb +103 -0
  205. data/lib/logstash/outputs/google_bigquery.rb +570 -0
  206. data/lib/logstash/outputs/google_cloud_storage.rb +431 -0
  207. data/lib/logstash/outputs/graphite.rb +143 -0
  208. data/lib/logstash/outputs/graphtastic.rb +185 -0
  209. data/lib/logstash/outputs/hipchat.rb +80 -0
  210. data/lib/logstash/outputs/http.rb +142 -0
  211. data/lib/logstash/outputs/irc.rb +80 -0
  212. data/lib/logstash/outputs/jira.rb +109 -0
  213. data/lib/logstash/outputs/juggernaut.rb +105 -0
  214. data/lib/logstash/outputs/librato.rb +146 -0
  215. data/lib/logstash/outputs/loggly.rb +93 -0
  216. data/lib/logstash/outputs/lumberjack.rb +51 -0
  217. data/lib/logstash/outputs/metriccatcher.rb +103 -0
  218. data/lib/logstash/outputs/mongodb.rb +81 -0
  219. data/lib/logstash/outputs/nagios.rb +119 -0
  220. data/lib/logstash/outputs/nagios_nsca.rb +123 -0
  221. data/lib/logstash/outputs/null.rb +18 -0
  222. data/lib/logstash/outputs/opentsdb.rb +101 -0
  223. data/lib/logstash/outputs/pagerduty.rb +79 -0
  224. data/lib/logstash/outputs/pipe.rb +132 -0
  225. data/lib/logstash/outputs/rabbitmq.rb +96 -0
  226. data/lib/logstash/outputs/rabbitmq/bunny.rb +135 -0
  227. data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +1 -0
  228. data/lib/logstash/outputs/rabbitmq/march_hare.rb +143 -0
  229. data/lib/logstash/outputs/redis.rb +245 -0
  230. data/lib/logstash/outputs/riak.rb +152 -0
  231. data/lib/logstash/outputs/riemann.rb +109 -0
  232. data/lib/logstash/outputs/s3.rb +356 -0
  233. data/lib/logstash/outputs/sns.rb +124 -0
  234. data/lib/logstash/outputs/solr_http.rb +78 -0
  235. data/lib/logstash/outputs/sqs.rb +141 -0
  236. data/lib/logstash/outputs/statsd.rb +116 -0
  237. data/lib/logstash/outputs/stdout.rb +53 -0
  238. data/lib/logstash/outputs/stomp.rb +67 -0
  239. data/lib/logstash/outputs/syslog.rb +145 -0
  240. data/lib/logstash/outputs/tcp.rb +145 -0
  241. data/lib/logstash/outputs/udp.rb +38 -0
  242. data/lib/logstash/outputs/websocket.rb +46 -0
  243. data/lib/logstash/outputs/websocket/app.rb +29 -0
  244. data/lib/logstash/outputs/websocket/pubsub.rb +45 -0
  245. data/lib/logstash/outputs/xmpp.rb +78 -0
  246. data/lib/logstash/outputs/zabbix.rb +108 -0
  247. data/lib/logstash/outputs/zeromq.rb +125 -0
  248. data/lib/logstash/pipeline.rb +286 -0
  249. data/lib/logstash/plugin.rb +150 -0
  250. data/lib/logstash/plugin_mixins/aws_config.rb +93 -0
  251. data/lib/logstash/program.rb +15 -0
  252. data/lib/logstash/runner.rb +238 -0
  253. data/lib/logstash/sized_queue.rb +8 -0
  254. data/lib/logstash/test.rb +183 -0
  255. data/lib/logstash/threadwatchdog.rb +37 -0
  256. data/lib/logstash/time_addon.rb +33 -0
  257. data/lib/logstash/util.rb +106 -0
  258. data/lib/logstash/util/buftok.rb +139 -0
  259. data/lib/logstash/util/charset.rb +39 -0
  260. data/lib/logstash/util/fieldreference.rb +50 -0
  261. data/lib/logstash/util/password.rb +25 -0
  262. data/lib/logstash/util/prctl.rb +11 -0
  263. data/lib/logstash/util/relp.rb +326 -0
  264. data/lib/logstash/util/require-helper.rb +18 -0
  265. data/lib/logstash/util/socket_peer.rb +7 -0
  266. data/lib/logstash/util/zenoss.rb +566 -0
  267. data/lib/logstash/util/zeromq.rb +47 -0
  268. data/lib/logstash/version.rb +6 -0
  269. data/locales/en.yml +170 -0
  270. data/logstash-event.gemspec +29 -0
  271. data/logstash.gemspec +128 -0
  272. data/patterns/firewalls +60 -0
  273. data/patterns/grok-patterns +91 -0
  274. data/patterns/haproxy +37 -0
  275. data/patterns/java +3 -0
  276. data/patterns/linux-syslog +14 -0
  277. data/patterns/mcollective +1 -0
  278. data/patterns/mcollective-patterns +4 -0
  279. data/patterns/nagios +108 -0
  280. data/patterns/postgresql +3 -0
  281. data/patterns/redis +3 -0
  282. data/patterns/ruby +2 -0
  283. data/pkg/build.sh +135 -0
  284. data/pkg/centos/after-install.sh +1 -0
  285. data/pkg/centos/before-install.sh +10 -0
  286. data/pkg/centos/before-remove.sh +11 -0
  287. data/pkg/centos/sysconfig +15 -0
  288. data/pkg/debian/after-install.sh +5 -0
  289. data/pkg/debian/before-install.sh +13 -0
  290. data/pkg/debian/before-remove.sh +13 -0
  291. data/pkg/debian/build.sh +34 -0
  292. data/pkg/debian/debian/README +6 -0
  293. data/pkg/debian/debian/changelog +17 -0
  294. data/pkg/debian/debian/compat +1 -0
  295. data/pkg/debian/debian/control +16 -0
  296. data/pkg/debian/debian/copyright +27 -0
  297. data/pkg/debian/debian/dirs +19 -0
  298. data/pkg/debian/debian/docs +0 -0
  299. data/pkg/debian/debian/logstash.default +39 -0
  300. data/pkg/debian/debian/logstash.init +201 -0
  301. data/pkg/debian/debian/logstash.install +1 -0
  302. data/pkg/debian/debian/logstash.logrotate +9 -0
  303. data/pkg/debian/debian/logstash.postinst +68 -0
  304. data/pkg/debian/debian/logstash.postrm +23 -0
  305. data/pkg/debian/debian/manpage.1.ex +59 -0
  306. data/pkg/debian/debian/preinst.ex +37 -0
  307. data/pkg/debian/debian/prerm.ex +40 -0
  308. data/pkg/debian/debian/release.conf +5 -0
  309. data/pkg/debian/debian/rules +80 -0
  310. data/pkg/debian/debian/watch.ex +22 -0
  311. data/pkg/logrotate.conf +8 -0
  312. data/pkg/logstash-web.default +41 -0
  313. data/pkg/logstash-web.sysv.debian +201 -0
  314. data/pkg/logstash-web.upstart.ubuntu +18 -0
  315. data/pkg/logstash.default +45 -0
  316. data/pkg/logstash.sysv.debian +202 -0
  317. data/pkg/logstash.sysv.redhat +158 -0
  318. data/pkg/logstash.upstart.ubuntu +20 -0
  319. data/pkg/rpm/SOURCES/logstash.conf +26 -0
  320. data/pkg/rpm/SOURCES/logstash.init +80 -0
  321. data/pkg/rpm/SOURCES/logstash.logrotate +8 -0
  322. data/pkg/rpm/SOURCES/logstash.sysconfig +3 -0
  323. data/pkg/rpm/SOURCES/logstash.wrapper +105 -0
  324. data/pkg/rpm/SPECS/logstash.spec +180 -0
  325. data/pkg/rpm/readme.md +4 -0
  326. data/pkg/ubuntu/after-install.sh +7 -0
  327. data/pkg/ubuntu/before-install.sh +12 -0
  328. data/pkg/ubuntu/before-remove.sh +13 -0
  329. data/pull_release_note.rb +25 -0
  330. data/require-analyze.rb +22 -0
  331. data/spec/README.md +14 -0
  332. data/spec/codecs/edn.rb +40 -0
  333. data/spec/codecs/edn_lines.rb +53 -0
  334. data/spec/codecs/graphite.rb +96 -0
  335. data/spec/codecs/json.rb +57 -0
  336. data/spec/codecs/json_lines.rb +51 -0
  337. data/spec/codecs/json_spooler.rb +43 -0
  338. data/spec/codecs/msgpack.rb +39 -0
  339. data/spec/codecs/multiline.rb +60 -0
  340. data/spec/codecs/oldlogstashjson.rb +55 -0
  341. data/spec/codecs/plain.rb +35 -0
  342. data/spec/codecs/spool.rb +35 -0
  343. data/spec/conditionals/test.rb +323 -0
  344. data/spec/config.rb +31 -0
  345. data/spec/event.rb +165 -0
  346. data/spec/examples/fail2ban.rb +28 -0
  347. data/spec/examples/graphite-input.rb +41 -0
  348. data/spec/examples/mysql-slow-query.rb +70 -0
  349. data/spec/examples/parse-apache-logs.rb +66 -0
  350. data/spec/examples/parse-haproxy-logs.rb +115 -0
  351. data/spec/examples/syslog.rb +48 -0
  352. data/spec/filters/alter.rb +96 -0
  353. data/spec/filters/anonymize.rb +189 -0
  354. data/spec/filters/checksum.rb +41 -0
  355. data/spec/filters/clone.rb +67 -0
  356. data/spec/filters/collate.rb +122 -0
  357. data/spec/filters/csv.rb +174 -0
  358. data/spec/filters/date.rb +285 -0
  359. data/spec/filters/date_performance.rb +31 -0
  360. data/spec/filters/dns.rb +159 -0
  361. data/spec/filters/drop.rb +19 -0
  362. data/spec/filters/elapsed.rb +294 -0
  363. data/spec/filters/environment.rb +43 -0
  364. data/spec/filters/geoip.rb +62 -0
  365. data/spec/filters/grep.rb +342 -0
  366. data/spec/filters/grok.rb +473 -0
  367. data/spec/filters/grok/timeout2.rb +56 -0
  368. data/spec/filters/grok/timeouts.rb +39 -0
  369. data/spec/filters/i18n.rb +25 -0
  370. data/spec/filters/json.rb +72 -0
  371. data/spec/filters/json_encode.rb +37 -0
  372. data/spec/filters/kv.rb +403 -0
  373. data/spec/filters/metrics.rb +212 -0
  374. data/spec/filters/multiline.rb +119 -0
  375. data/spec/filters/mutate.rb +180 -0
  376. data/spec/filters/noop.rb +221 -0
  377. data/spec/filters/prune.rb +441 -0
  378. data/spec/filters/punct.rb +18 -0
  379. data/spec/filters/railsparallelrequest.rb +112 -0
  380. data/spec/filters/range.rb +169 -0
  381. data/spec/filters/split.rb +58 -0
  382. data/spec/filters/translate.rb +70 -0
  383. data/spec/filters/unique.rb +25 -0
  384. data/spec/filters/useragent.rb +42 -0
  385. data/spec/filters/xml.rb +157 -0
  386. data/spec/inputs/file.rb +107 -0
  387. data/spec/inputs/gelf.rb +52 -0
  388. data/spec/inputs/generator.rb +30 -0
  389. data/spec/inputs/imap.rb +60 -0
  390. data/spec/inputs/redis.rb +63 -0
  391. data/spec/inputs/relp.rb +70 -0
  392. data/spec/inputs/tcp.rb +101 -0
  393. data/spec/jar.rb +21 -0
  394. data/spec/outputs/csv.rb +266 -0
  395. data/spec/outputs/elasticsearch.rb +161 -0
  396. data/spec/outputs/elasticsearch_http.rb +240 -0
  397. data/spec/outputs/email.rb +173 -0
  398. data/spec/outputs/file.rb +82 -0
  399. data/spec/outputs/graphite.rb +236 -0
  400. data/spec/outputs/redis.rb +127 -0
  401. data/spec/speed.rb +20 -0
  402. data/spec/sqlite-test.rb +81 -0
  403. data/spec/support/LOGSTASH-733.rb +21 -0
  404. data/spec/support/LOGSTASH-820.rb +25 -0
  405. data/spec/support/akamai-grok.rb +26 -0
  406. data/spec/support/date-http.rb +17 -0
  407. data/spec/support/postwait1.rb +26 -0
  408. data/spec/support/pull375.rb +21 -0
  409. data/spec/test_utils.rb +125 -0
  410. data/spec/util/fieldeval_spec.rb +44 -0
  411. data/test/jenkins/config.xml.erb +74 -0
  412. data/test/jenkins/create-jobs.rb +23 -0
  413. data/test/jenkins/generatorjob.config.xml +66 -0
  414. data/tools/Gemfile +14 -0
  415. data/tools/Gemfile.jruby-1.9.lock +322 -0
  416. data/tools/Gemfile.rbx-2.1.lock +516 -0
  417. data/tools/Gemfile.ruby-1.9.1.lock +310 -0
  418. data/tools/Gemfile.ruby-2.0.0.lock +310 -0
  419. metadata +629 -0
@@ -0,0 +1,80 @@
1
+ #!/bin/bash
2
+ # chkconfig: - 80 15
3
+ ### BEGIN INIT INFO
4
+ # Provides: logstash
5
+ # Required-Start: $all
6
+ # Required-Stop: $all
7
+ # Default-Start:
8
+ # Default-Stop: 0 1 6
9
+ # Short-Description: Starts logstash
10
+ # Description: Logstash agent
11
+ ### END INIT INFO
12
+
13
+ # Source function library.
14
+ . /etc/rc.d/init.d/functions
15
+
16
+ NAME=@@@NAME@@@
17
+
18
+ [ -f /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME
19
+
20
+ LOGSTASH_USER=${NAME}
21
+ DAEMON="@@@DAEMON@@@/${NAME}"
22
+ SERVICE=agent
23
+
24
+ PID_FILE=${PIDFILE:-@@@PIDDIR@@@/${NAME}.pid}
25
+ LOCK_FILE=${LOCKFILE:-@@@LOCKFILE@@@}
26
+ LOG_FILE=${LOGFILE:-@@@LOGDIR@@@/${NAME}.log}
27
+
28
+ LOGSTASH_PATH_CONF=${LOGSTASH_PATH_CONF:-@@@CONFDIR@@@}
29
+ LOGSTASH_LOGLEVEL=${LOGSTASH_LOGLEVEL:-"warn"}
30
+
31
+ DAEMON_OPTS="\
32
+ -P ${PID_FILE} \
33
+ -l ${LOG_FILE} \
34
+ -f ${LOGSTASH_PATH_CONF} \
35
+ -v $LOGSTASH_LOGLEVEL \
36
+ "
37
+
38
+ start() {
39
+ echo -n $"Starting ${NAME}: "
40
+ export JAVA_OPTS="$JAVA_OPTS $LOGSTASH_JAVA_OPTS"
41
+ daemon --pidfile=${PID_FILE} --user $LOGSTASH_USER $DAEMON $SERVICE $DAEMON_OPTS
42
+ RETVAL=$?
43
+ echo
44
+ [ $RETVAL -eq 0 ] && touch $LOCK_FILE
45
+ return $RETVAL
46
+ }
47
+
48
+ stop() {
49
+
50
+ echo -n $"Stopping ${NAME}: "
51
+ killproc -p ${PID_FILE} -d 10 $DAEMON
52
+ RETVAL=$?
53
+ echo
54
+ [ $RETVAL = 0 ] && rm -f ${LOCK_FILE} ${PID_FILE}
55
+ return $RETVAL
56
+ }
57
+
58
+ case "$1" in
59
+ start)
60
+ start
61
+ ;;
62
+ stop)
63
+ stop
64
+ ;;
65
+ status)
66
+ status -p ${PID_FILE} $DAEMON
67
+ RETVAL=$?
68
+ ;;
69
+ restart|force-reload)
70
+ stop
71
+ start
72
+ ;;
73
+ *)
74
+ N=/etc/init.d/${NAME}
75
+ echo "Usage: $N {start|stop|restart|force-reload}" >&2
76
+ RETVAL=2
77
+ ;;
78
+ esac
79
+
80
+ exit $RETVAL
@@ -0,0 +1,8 @@
1
+ /var/log/logstash/*.log {
2
+ daily
3
+ rotate 7
4
+ copytruncate
5
+ compress
6
+ missingok
7
+ notifempty
8
+ }
@@ -0,0 +1,3 @@
1
+ #LOGSTASH_LOGFILE=@@@LOGDIR@@@/@@@NAME@@@.log
2
+ #LOGSTASH_PATH_CONF=@@@CONFDIR@@@
3
+ #LOGSTASH_JAVA_OPTS="-Djava.io.tmpdir=/opt/@@@NAME@@@/tmp"
@@ -0,0 +1,105 @@
1
+ #!/bin/sh
2
+ SCRIPT=$0
3
+
4
+ if [ -x "$JAVA_HOME/bin/java" ]; then
5
+ JAVA=$JAVA_HOME/bin/java
6
+ else
7
+ JAVA=`which java`
8
+ fi
9
+
10
+ LOGSTASH_JAR="@@@JARPATH@@@/@@@NAME@@@.jar"
11
+ if [ ! -f $LOGSTASH_JAR ]
12
+ then
13
+ echo "jar file is not found."
14
+ exit 99
15
+ fi
16
+
17
+ function usage() {
18
+ echo "Usage: ${SCRIPT} SERVICE OPTIONS"
19
+ echo " SERVICE: agent, web"
20
+ echo " OPTIONS:"
21
+ echo -e " -f, --config CONFIGFILE (required)\tLoad the logstash config from a specific file or directory."
22
+ echo -e " \t\tIf a direcory is given instead of a file, all files in that directory will be concatonated in lexicographical order and then parsed as a single config file."
23
+ echo -e " -P, --pidfile PIDFILE\t\tPID file path."
24
+ echo -e " -l, --logfile LOGFILE\t\tLogfile path."
25
+ echo -e " -v, --verbose [info, debug]\t\tEnables more verbose logging"
26
+ }
27
+
28
+ function run_service() {
29
+ service=$1
30
+ config=$2
31
+ pidfile=$3
32
+ logfile=$4
33
+ verbose=$5
34
+
35
+ if [ "x$logfile" == "x" ]; then
36
+ exec "$JAVA" $JAVA_OPTS -jar $LOGSTASH_JAR $service $verbose -f "$config" &
37
+ rs=$?
38
+ else
39
+ exec "$JAVA" $JAVA_OPTS -jar $LOGSTASH_JAR $service $verbose -f "$config" -l "$logfile" 2>&1 >> $logfile &
40
+ rs=$?
41
+ [ $rs -eq 0 -a "x$pidfile" != "x" ] && printf '%d' $! > "$pidfile"
42
+ fi
43
+
44
+ return $rs
45
+ }
46
+
47
+ service=$1; shift
48
+
49
+ if [ "${service}" != "agent" -a "${service}" != "web" ]
50
+ then
51
+ echo "ERROR: no such service \`${service}'. Available services are: agent, web"
52
+ usage
53
+ exit 99
54
+ fi
55
+
56
+ while test $# -gt 0
57
+ do
58
+ case "$1" in
59
+ -V | --version )
60
+ "$JAVA" -jar $LOGSTASH_JAR --version
61
+ exit 0
62
+ ;;
63
+ -f | --config )
64
+ config="$2"
65
+ if [ ! -f "$config" -a ! -d "$config" ]; then
66
+ echo "ERROR: config file or directory \`$config' does not exist."
67
+ usage
68
+ exit 1
69
+ fi
70
+ shift 2
71
+ ;;
72
+ -P | --pidfile )
73
+ pidfile="$2"
74
+ shift 2
75
+ ;;
76
+ -l | --logfile )
77
+ logfile="$2"
78
+ shift 2
79
+ ;;
80
+ -v | --verbose )
81
+ if [ "$2" == "debug" ]; then
82
+ verbose="-vv"
83
+ elif [ "$2" == "info" ]; then
84
+ verbose="-v"
85
+ fi
86
+ shift 2
87
+ ;;
88
+ -h | --help )
89
+ usage
90
+ exit 0
91
+ ;;
92
+ * )
93
+ break
94
+ esac
95
+ done
96
+
97
+ if [ "x$config" == "x" ]; then
98
+ echo "ERROR: config file is required."
99
+ usage
100
+ exit 1
101
+ fi
102
+
103
+ run_service "$service" "$config" "$pidfile" "$logfile" "$verbose"
104
+
105
+ exit $?
@@ -0,0 +1,180 @@
1
+ %define debug_package %{nil}_bindir}
2
+ %define base_install_dir %{_javadir}{%name}
3
+
4
+ %global bindir %{_bindir}
5
+ %global confdir %{_sysconfdir}/%{name}
6
+ %global jarpath %{_javadir}
7
+ %global lockfile %{_localstatedir}/lock/subsys/%{name}
8
+ %global logdir %{_localstatedir}/log/%{name}
9
+ %global piddir %{_localstatedir}/run/%{name}
10
+ %global sysconfigdir %{_sysconfdir}/sysconfig
11
+
12
+ Name: logstash
13
+ Version: 1.2.2
14
+ Release: 2%{?dist}
15
+ Summary: A tool for managing events and logs
16
+
17
+ Group: System Environment/Daemons
18
+ License: ASL 2.0
19
+ URL: http://logstash.net
20
+ Source0: https://logstash.objects.dreamhost.com/release/%{name}-%{version}-flatjar.jar
21
+ Source1: logstash.wrapper
22
+ Source2: logstash.logrotate
23
+ Source3: logstash.init
24
+ Source4: logstash.sysconfig
25
+ Source5: logstash.conf
26
+ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
27
+ BuildArch: x86_64 i386 i686
28
+
29
+ Requires: java
30
+ Requires: jpackage-utils
31
+
32
+ Requires(post): chkconfig initscripts
33
+ Requires(pre): chkconfig initscripts
34
+ Requires(pre): shadow-utils
35
+
36
+ %description
37
+ A tool for managing events and logs.
38
+
39
+ %prep
40
+ %build
41
+
42
+ %install
43
+ rm -rf $RPM_BUILD_ROOT
44
+
45
+ # JAR file
46
+ %{__mkdir} -p %{buildroot}%{_javadir}
47
+ %{__install} -p -m 644 %{SOURCE0} %{buildroot}%{jarpath}/%{name}.jar
48
+
49
+ # Config
50
+ %{__mkdir} -p %{buildroot}%{confdir}
51
+ %{__install} -m 644 %{SOURCE5} %{buildroot}%{confdir}/%{name}.conf
52
+
53
+ # Wrapper script
54
+ %{__mkdir} -p %{buildroot}%{_bindir}
55
+ %{__install} -m 755 %{SOURCE1} %{buildroot}%{bindir}/%{name}
56
+
57
+ %{__sed} -i \
58
+ -e "s|@@@NAME@@@|%{name}|g" \
59
+ -e "s|@@@JARPATH@@@|%{jarpath}|g" \
60
+ %{buildroot}%{bindir}/%{name}
61
+
62
+ # Logs
63
+ %{__mkdir} -p %{buildroot}%{logdir}
64
+ %{__install} -D -m 644 %{SOURCE2} %{buildroot}%{_sysconfdir}/logrotate.d/%{name}
65
+
66
+ # Misc
67
+ %{__mkdir} -p %{buildroot}%{piddir}
68
+
69
+ # sysconfig and init
70
+ %{__mkdir} -p %{buildroot}%{_initddir}
71
+ %{__mkdir} -p %{buildroot}%{_sysconfdir}/sysconfig
72
+ %{__install} -m 755 %{SOURCE3} %{buildroot}%{_initddir}/%{name}
73
+ %{__install} -m 644 %{SOURCE4} %{buildroot}%{sysconfigdir}/%{name}
74
+
75
+ %{__sed} -i \
76
+ -e "s|@@@NAME@@@|%{name}|g" \
77
+ -e "s|@@@DAEMON@@@|%{bindir}|g" \
78
+ -e "s|@@@CONFDIR@@@|%{confdir}|g" \
79
+ -e "s|@@@LOCKFILE@@@|%{lockfile}|g" \
80
+ -e "s|@@@LOGDIR@@@|%{logdir}|g" \
81
+ -e "s|@@@PIDDIR@@@|%{piddir}|g" \
82
+ %{buildroot}%{_initddir}/%{name}
83
+
84
+ %{__sed} -i \
85
+ -e "s|@@@NAME@@@|%{name}|g" \
86
+ -e "s|@@@CONFDIR@@@|%{confdir}|g" \
87
+ -e "s|@@@LOGDIR@@@|%{logdir}|g" \
88
+ -e "s|@@@PLUGINDIR@@@|%{_datadir}|g" \
89
+ %{buildroot}%{sysconfigdir}/%{name}
90
+
91
+ %pre
92
+ # create logstash group
93
+ if ! getent group logstash >/dev/null; then
94
+ groupadd -r logstash
95
+ fi
96
+
97
+ # create logstash user
98
+ if ! getent passwd logstash >/dev/null; then
99
+ useradd -r -g logstash -d %{_javadir}/%{name} \
100
+ -s /sbin/nologin -c "You know, for search" logstash
101
+ fi
102
+
103
+ %post
104
+ /sbin/chkconfig --add logstash
105
+
106
+ %preun
107
+ if [ $1 -eq 0 ]; then
108
+ /sbin/service logstash stop >/dev/null 2>&1
109
+ /sbin/chkconfig --del logstash
110
+ fi
111
+
112
+ %clean
113
+ rm -rf $RPM_BUILD_ROOT
114
+
115
+ %files
116
+ %defattr(-,root,root,-)
117
+ # JAR file
118
+ %{_javadir}/%{name}.jar
119
+
120
+ # Config
121
+ %config(noreplace) %{confdir}/
122
+
123
+ # Wrapper script
124
+ %{bindir}/*
125
+
126
+
127
+ # Logrotate
128
+ %config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
129
+
130
+ # Sysconfig and init
131
+ %{_initddir}/%{name}
132
+ %config(noreplace) %{sysconfigdir}/*
133
+
134
+ %defattr(-,%{name},%{name},-)
135
+ %dir %{logdir}/
136
+ %dir %{piddir}/
137
+
138
+ %changelog
139
+ * Sun Oct 27 2013 <sbagmeijer@ulyaoth.asia> - 1.2.2-2
140
+ - Fixed the bogus date warning
141
+
142
+ * Wed Oct 23 2013 <sjir@basefarm.no> - 1.2.2-1
143
+ - Update logstash version to 1.2.2-1
144
+
145
+ * Fri Sep 27 2013 <sbagmeijer@ulyaoth.asia> - 1.2.1-1
146
+ - Building for Fedora 18, 19 and 20-alpha.
147
+ - Added Source5 for logstash.conf
148
+ - Fixed the BuildArch so it can be build for i386
149
+
150
+ * Mon Sep 16 2013 <sjir@basefarm.se> - 1.2.1
151
+ - Updated version to the new 1.2.1
152
+ - Removed everything related to plugins as it no longer works.
153
+
154
+ * Wed Sep 04 2013 <sjir@basefarm.se> - 1.2.0
155
+ - Updated version to the new 1.2.0.
156
+ - Fixed a problem with the init.d script not working correctly.
157
+
158
+ * Fri Jun 14 2013 <sjir@basefarm.se> - 1.1.13-1
159
+ - Updated version to the new 1.1.13-1 and fixed some minor issues with directory structure.
160
+
161
+ * Mon May 6 2013 <sjir@basefarm.se> - 1.1.10-3
162
+ - Changed from logstash flatjar to the monolith as flatjar is not working correctly yet.
163
+
164
+ * Fri Apr 19 2013 <sjir@basefarm.se> - 1.1.10-2
165
+ - Fixed a bug
166
+
167
+ * Fri Apr 19 2013 <sjir@basefarm.se> - 1.1.10-1
168
+ - Added fixes to support RHEL6
169
+ - Update logstash version to 1.1.10
170
+
171
+ * Sun Mar 17 2013 Richard Pijnenburg <richard@ispavailability.com> - 1.1.9-2
172
+ - Update init script
173
+ - Create patterns dir in correct place
174
+
175
+ * Fri Feb 1 2013 Richard Pijnenburg <richard@ispavailability.com> - 1.1.9-1
176
+ - Update to latest stable release.
177
+ - New init script
178
+
179
+ * Fri May 4 2012 Maksim Horbul <max@gorbul.net> - 1.1.0-1
180
+ - Initial package
@@ -0,0 +1,4 @@
1
+ # RPM build
2
+
3
+ spectool -g SPECS/logstash.spec
4
+ rpmbuild -bb SPECS/logstash.spec
@@ -0,0 +1,7 @@
1
+ #!/bin/sh
2
+
3
+ chown -R logstash:logstash /opt/logstash
4
+ chown logstash /var/log/logstash
5
+ chown logstash:logstash /var/lib/logstash
6
+
7
+ ln -sf /opt/logstash/logstash.jar /var/lib/logstash/logstash.jar
@@ -0,0 +1,12 @@
1
+ #!/bin/sh
2
+
3
+ # create logstash group
4
+ if ! getent group logstash >/dev/null; then
5
+ groupadd -r logstash
6
+ fi
7
+
8
+ # create logstash user
9
+ if ! getent passwd logstash >/dev/null; then
10
+ useradd -M -r -g logstash -d /var/lib/logstash \
11
+ -s /sbin/nologin -c "LogStash Service User" logstash
12
+ fi
@@ -0,0 +1,13 @@
1
+ #!/bin/sh
2
+
3
+ if [ $1 == "remove" ]; then
4
+ stop logstash >/dev/null 2>&1 || true
5
+
6
+ if getent passwd logstash >/dev/null ; then
7
+ userdel logstash
8
+ fi
9
+
10
+ if getent group logstash >/dev/null ; then
11
+ groupdel logstash
12
+ fi
13
+ fi
@@ -0,0 +1,25 @@
1
+ require "octokit"
2
+
3
+
4
+ @repository= "logstash/logstash"
5
+ @releaseNote= "releaseNote.html"
6
+
7
+ #Last release == last tag
8
+ lastReleaseSha = Octokit.tags(@repository).first.commit.sha
9
+
10
+ currentReleaseSha ="HEAD"
11
+
12
+ #Collect PR Merge in a file
13
+ File.open(@releaseNote, "a") do |f|
14
+ f.puts "<h2>Merged pull request</h2>"
15
+ f.puts "<ul>"
16
+ Octokit.compare(@repository, lastReleaseSha, currentReleaseSha).commits.each do |commit|
17
+ if commit.commit.message.start_with?("Merge pull")
18
+ scan_re = Regexp.new(/^Merge pull request #(\d+) from ([^\/]+)\/.*\n\n(.*)/)
19
+ commit.commit.message.scan(scan_re) do |pullNumber, user, summary|
20
+ f.puts "<li><a href='https://github.com/logstash/logstash/pull/#{pullNumber}'>Pull ##{pullNumber}<a> by #{user}: #{summary}</li>"
21
+ end
22
+ end
23
+ end
24
+ f.puts "</ul>"
25
+ end