logstash-lib 1.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (419) hide show
  1. data/.gitignore +24 -0
  2. data/.tailor +8 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG +1185 -0
  5. data/CONTRIBUTING.md +61 -0
  6. data/CONTRIBUTORS +79 -0
  7. data/LICENSE +14 -0
  8. data/Makefile +460 -0
  9. data/README.md +120 -0
  10. data/STYLE.md +96 -0
  11. data/bin/logstash +37 -0
  12. data/bin/logstash-test +4 -0
  13. data/bin/logstash-web +4 -0
  14. data/bin/logstash.lib.sh +78 -0
  15. data/bot/check_pull_changelog.rb +89 -0
  16. data/docs/configuration.md +260 -0
  17. data/docs/docgen.rb +242 -0
  18. data/docs/extending/example-add-a-new-filter.md +121 -0
  19. data/docs/extending/index.md +91 -0
  20. data/docs/flags.md +43 -0
  21. data/docs/generate_index.rb +28 -0
  22. data/docs/index.html.erb +56 -0
  23. data/docs/learn.md +46 -0
  24. data/docs/life-of-an-event.md +109 -0
  25. data/docs/logging-tool-comparisons.md +60 -0
  26. data/docs/plugin-doc.html.erb +91 -0
  27. data/docs/plugin-milestones.md +41 -0
  28. data/docs/plugin-synopsis.html.erb +24 -0
  29. data/docs/release-engineering.md +46 -0
  30. data/docs/release-test-results.md +14 -0
  31. data/docs/repositories.md +35 -0
  32. data/docs/tutorials/10-minute-walkthrough/apache-elasticsearch.conf +35 -0
  33. data/docs/tutorials/10-minute-walkthrough/apache-parse.conf +33 -0
  34. data/docs/tutorials/10-minute-walkthrough/apache_log.1 +1 -0
  35. data/docs/tutorials/10-minute-walkthrough/apache_log.2.bz2 +0 -0
  36. data/docs/tutorials/10-minute-walkthrough/hello-search.conf +25 -0
  37. data/docs/tutorials/10-minute-walkthrough/hello.conf +16 -0
  38. data/docs/tutorials/10-minute-walkthrough/index.md +124 -0
  39. data/docs/tutorials/10-minute-walkthrough/step-5-output.txt +17 -0
  40. data/docs/tutorials/getting-started-centralized-overview-diagram.png +0 -0
  41. data/docs/tutorials/getting-started-centralized-overview-diagram.xml +1 -0
  42. data/docs/tutorials/getting-started-centralized.md +217 -0
  43. data/docs/tutorials/getting-started-simple.md +200 -0
  44. data/docs/tutorials/just-enough-rabbitmq-for-logstash.md +201 -0
  45. data/docs/tutorials/media/frontend-response-codes.png +0 -0
  46. data/docs/tutorials/metrics-from-logs.md +84 -0
  47. data/docs/tutorials/zeromq.md +118 -0
  48. data/extract_services.rb +29 -0
  49. data/gembag.rb +64 -0
  50. data/lib/logstash-event.rb +2 -0
  51. data/lib/logstash.rb +4 -0
  52. data/lib/logstash/JRUBY-6970-openssl.rb +22 -0
  53. data/lib/logstash/JRUBY-6970.rb +102 -0
  54. data/lib/logstash/agent.rb +305 -0
  55. data/lib/logstash/certs/cacert.pem +3895 -0
  56. data/lib/logstash/codecs/base.rb +49 -0
  57. data/lib/logstash/codecs/compress_spooler.rb +50 -0
  58. data/lib/logstash/codecs/dots.rb +18 -0
  59. data/lib/logstash/codecs/edn.rb +28 -0
  60. data/lib/logstash/codecs/edn_lines.rb +36 -0
  61. data/lib/logstash/codecs/fluent.rb +55 -0
  62. data/lib/logstash/codecs/graphite.rb +114 -0
  63. data/lib/logstash/codecs/json.rb +41 -0
  64. data/lib/logstash/codecs/json_lines.rb +52 -0
  65. data/lib/logstash/codecs/json_spooler.rb +22 -0
  66. data/lib/logstash/codecs/line.rb +58 -0
  67. data/lib/logstash/codecs/msgpack.rb +43 -0
  68. data/lib/logstash/codecs/multiline.rb +189 -0
  69. data/lib/logstash/codecs/netflow.rb +342 -0
  70. data/lib/logstash/codecs/netflow/util.rb +212 -0
  71. data/lib/logstash/codecs/noop.rb +19 -0
  72. data/lib/logstash/codecs/oldlogstashjson.rb +56 -0
  73. data/lib/logstash/codecs/plain.rb +48 -0
  74. data/lib/logstash/codecs/rubydebug.rb +22 -0
  75. data/lib/logstash/codecs/spool.rb +38 -0
  76. data/lib/logstash/config/Makefile +4 -0
  77. data/lib/logstash/config/config_ast.rb +380 -0
  78. data/lib/logstash/config/file.rb +39 -0
  79. data/lib/logstash/config/grammar.rb +3504 -0
  80. data/lib/logstash/config/grammar.treetop +241 -0
  81. data/lib/logstash/config/mixin.rb +464 -0
  82. data/lib/logstash/config/registry.rb +13 -0
  83. data/lib/logstash/config/test.conf +18 -0
  84. data/lib/logstash/errors.rb +10 -0
  85. data/lib/logstash/event.rb +262 -0
  86. data/lib/logstash/filters/advisor.rb +178 -0
  87. data/lib/logstash/filters/alter.rb +173 -0
  88. data/lib/logstash/filters/anonymize.rb +93 -0
  89. data/lib/logstash/filters/base.rb +190 -0
  90. data/lib/logstash/filters/checksum.rb +50 -0
  91. data/lib/logstash/filters/cidr.rb +76 -0
  92. data/lib/logstash/filters/cipher.rb +145 -0
  93. data/lib/logstash/filters/clone.rb +35 -0
  94. data/lib/logstash/filters/collate.rb +114 -0
  95. data/lib/logstash/filters/csv.rb +94 -0
  96. data/lib/logstash/filters/date.rb +244 -0
  97. data/lib/logstash/filters/dns.rb +201 -0
  98. data/lib/logstash/filters/drop.rb +32 -0
  99. data/lib/logstash/filters/elapsed.rb +256 -0
  100. data/lib/logstash/filters/elasticsearch.rb +73 -0
  101. data/lib/logstash/filters/environment.rb +27 -0
  102. data/lib/logstash/filters/extractnumbers.rb +84 -0
  103. data/lib/logstash/filters/gelfify.rb +52 -0
  104. data/lib/logstash/filters/geoip.rb +145 -0
  105. data/lib/logstash/filters/grep.rb +153 -0
  106. data/lib/logstash/filters/grok.rb +425 -0
  107. data/lib/logstash/filters/grokdiscovery.rb +75 -0
  108. data/lib/logstash/filters/i18n.rb +51 -0
  109. data/lib/logstash/filters/json.rb +90 -0
  110. data/lib/logstash/filters/json_encode.rb +52 -0
  111. data/lib/logstash/filters/kv.rb +232 -0
  112. data/lib/logstash/filters/metaevent.rb +68 -0
  113. data/lib/logstash/filters/metrics.rb +237 -0
  114. data/lib/logstash/filters/multiline.rb +241 -0
  115. data/lib/logstash/filters/mutate.rb +399 -0
  116. data/lib/logstash/filters/noop.rb +21 -0
  117. data/lib/logstash/filters/prune.rb +149 -0
  118. data/lib/logstash/filters/punct.rb +32 -0
  119. data/lib/logstash/filters/railsparallelrequest.rb +86 -0
  120. data/lib/logstash/filters/range.rb +142 -0
  121. data/lib/logstash/filters/ruby.rb +42 -0
  122. data/lib/logstash/filters/sleep.rb +111 -0
  123. data/lib/logstash/filters/split.rb +64 -0
  124. data/lib/logstash/filters/sumnumbers.rb +73 -0
  125. data/lib/logstash/filters/syslog_pri.rb +107 -0
  126. data/lib/logstash/filters/translate.rb +121 -0
  127. data/lib/logstash/filters/unique.rb +29 -0
  128. data/lib/logstash/filters/urldecode.rb +57 -0
  129. data/lib/logstash/filters/useragent.rb +112 -0
  130. data/lib/logstash/filters/uuid.rb +58 -0
  131. data/lib/logstash/filters/xml.rb +139 -0
  132. data/lib/logstash/filters/zeromq.rb +123 -0
  133. data/lib/logstash/filterworker.rb +122 -0
  134. data/lib/logstash/inputs/base.rb +125 -0
  135. data/lib/logstash/inputs/collectd.rb +306 -0
  136. data/lib/logstash/inputs/drupal_dblog.rb +323 -0
  137. data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb +66 -0
  138. data/lib/logstash/inputs/elasticsearch.rb +140 -0
  139. data/lib/logstash/inputs/eventlog.rb +129 -0
  140. data/lib/logstash/inputs/eventlog/racob_fix.rb +44 -0
  141. data/lib/logstash/inputs/exec.rb +69 -0
  142. data/lib/logstash/inputs/file.rb +146 -0
  143. data/lib/logstash/inputs/ganglia.rb +127 -0
  144. data/lib/logstash/inputs/ganglia/gmondpacket.rb +146 -0
  145. data/lib/logstash/inputs/ganglia/xdr.rb +327 -0
  146. data/lib/logstash/inputs/gelf.rb +138 -0
  147. data/lib/logstash/inputs/gemfire.rb +222 -0
  148. data/lib/logstash/inputs/generator.rb +97 -0
  149. data/lib/logstash/inputs/graphite.rb +41 -0
  150. data/lib/logstash/inputs/heroku.rb +51 -0
  151. data/lib/logstash/inputs/imap.rb +136 -0
  152. data/lib/logstash/inputs/irc.rb +84 -0
  153. data/lib/logstash/inputs/log4j.rb +136 -0
  154. data/lib/logstash/inputs/lumberjack.rb +53 -0
  155. data/lib/logstash/inputs/pipe.rb +57 -0
  156. data/lib/logstash/inputs/rabbitmq.rb +126 -0
  157. data/lib/logstash/inputs/rabbitmq/bunny.rb +118 -0
  158. data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb +1 -0
  159. data/lib/logstash/inputs/rabbitmq/march_hare.rb +129 -0
  160. data/lib/logstash/inputs/redis.rb +263 -0
  161. data/lib/logstash/inputs/relp.rb +106 -0
  162. data/lib/logstash/inputs/s3.rb +279 -0
  163. data/lib/logstash/inputs/snmptrap.rb +87 -0
  164. data/lib/logstash/inputs/sqlite.rb +185 -0
  165. data/lib/logstash/inputs/sqs.rb +172 -0
  166. data/lib/logstash/inputs/stdin.rb +46 -0
  167. data/lib/logstash/inputs/stomp.rb +84 -0
  168. data/lib/logstash/inputs/syslog.rb +237 -0
  169. data/lib/logstash/inputs/tcp.rb +231 -0
  170. data/lib/logstash/inputs/threadable.rb +18 -0
  171. data/lib/logstash/inputs/twitter.rb +82 -0
  172. data/lib/logstash/inputs/udp.rb +81 -0
  173. data/lib/logstash/inputs/unix.rb +163 -0
  174. data/lib/logstash/inputs/varnishlog.rb +48 -0
  175. data/lib/logstash/inputs/websocket.rb +50 -0
  176. data/lib/logstash/inputs/wmi.rb +72 -0
  177. data/lib/logstash/inputs/xmpp.rb +81 -0
  178. data/lib/logstash/inputs/zenoss.rb +143 -0
  179. data/lib/logstash/inputs/zeromq.rb +165 -0
  180. data/lib/logstash/kibana.rb +113 -0
  181. data/lib/logstash/loadlibs.rb +9 -0
  182. data/lib/logstash/logging.rb +89 -0
  183. data/lib/logstash/monkeypatches-for-bugs.rb +2 -0
  184. data/lib/logstash/monkeypatches-for-debugging.rb +47 -0
  185. data/lib/logstash/monkeypatches-for-performance.rb +66 -0
  186. data/lib/logstash/multiqueue.rb +53 -0
  187. data/lib/logstash/namespace.rb +16 -0
  188. data/lib/logstash/outputs/base.rb +120 -0
  189. data/lib/logstash/outputs/boundary.rb +116 -0
  190. data/lib/logstash/outputs/circonus.rb +78 -0
  191. data/lib/logstash/outputs/cloudwatch.rb +351 -0
  192. data/lib/logstash/outputs/csv.rb +55 -0
  193. data/lib/logstash/outputs/datadog.rb +93 -0
  194. data/lib/logstash/outputs/datadog_metrics.rb +123 -0
  195. data/lib/logstash/outputs/elasticsearch.rb +332 -0
  196. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +44 -0
  197. data/lib/logstash/outputs/elasticsearch_http.rb +256 -0
  198. data/lib/logstash/outputs/elasticsearch_river.rb +214 -0
  199. data/lib/logstash/outputs/email.rb +299 -0
  200. data/lib/logstash/outputs/exec.rb +40 -0
  201. data/lib/logstash/outputs/file.rb +180 -0
  202. data/lib/logstash/outputs/ganglia.rb +75 -0
  203. data/lib/logstash/outputs/gelf.rb +208 -0
  204. data/lib/logstash/outputs/gemfire.rb +103 -0
  205. data/lib/logstash/outputs/google_bigquery.rb +570 -0
  206. data/lib/logstash/outputs/google_cloud_storage.rb +431 -0
  207. data/lib/logstash/outputs/graphite.rb +143 -0
  208. data/lib/logstash/outputs/graphtastic.rb +185 -0
  209. data/lib/logstash/outputs/hipchat.rb +80 -0
  210. data/lib/logstash/outputs/http.rb +142 -0
  211. data/lib/logstash/outputs/irc.rb +80 -0
  212. data/lib/logstash/outputs/jira.rb +109 -0
  213. data/lib/logstash/outputs/juggernaut.rb +105 -0
  214. data/lib/logstash/outputs/librato.rb +146 -0
  215. data/lib/logstash/outputs/loggly.rb +93 -0
  216. data/lib/logstash/outputs/lumberjack.rb +51 -0
  217. data/lib/logstash/outputs/metriccatcher.rb +103 -0
  218. data/lib/logstash/outputs/mongodb.rb +81 -0
  219. data/lib/logstash/outputs/nagios.rb +119 -0
  220. data/lib/logstash/outputs/nagios_nsca.rb +123 -0
  221. data/lib/logstash/outputs/null.rb +18 -0
  222. data/lib/logstash/outputs/opentsdb.rb +101 -0
  223. data/lib/logstash/outputs/pagerduty.rb +79 -0
  224. data/lib/logstash/outputs/pipe.rb +132 -0
  225. data/lib/logstash/outputs/rabbitmq.rb +96 -0
  226. data/lib/logstash/outputs/rabbitmq/bunny.rb +135 -0
  227. data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +1 -0
  228. data/lib/logstash/outputs/rabbitmq/march_hare.rb +143 -0
  229. data/lib/logstash/outputs/redis.rb +245 -0
  230. data/lib/logstash/outputs/riak.rb +152 -0
  231. data/lib/logstash/outputs/riemann.rb +109 -0
  232. data/lib/logstash/outputs/s3.rb +356 -0
  233. data/lib/logstash/outputs/sns.rb +124 -0
  234. data/lib/logstash/outputs/solr_http.rb +78 -0
  235. data/lib/logstash/outputs/sqs.rb +141 -0
  236. data/lib/logstash/outputs/statsd.rb +116 -0
  237. data/lib/logstash/outputs/stdout.rb +53 -0
  238. data/lib/logstash/outputs/stomp.rb +67 -0
  239. data/lib/logstash/outputs/syslog.rb +145 -0
  240. data/lib/logstash/outputs/tcp.rb +145 -0
  241. data/lib/logstash/outputs/udp.rb +38 -0
  242. data/lib/logstash/outputs/websocket.rb +46 -0
  243. data/lib/logstash/outputs/websocket/app.rb +29 -0
  244. data/lib/logstash/outputs/websocket/pubsub.rb +45 -0
  245. data/lib/logstash/outputs/xmpp.rb +78 -0
  246. data/lib/logstash/outputs/zabbix.rb +108 -0
  247. data/lib/logstash/outputs/zeromq.rb +125 -0
  248. data/lib/logstash/pipeline.rb +286 -0
  249. data/lib/logstash/plugin.rb +150 -0
  250. data/lib/logstash/plugin_mixins/aws_config.rb +93 -0
  251. data/lib/logstash/program.rb +15 -0
  252. data/lib/logstash/runner.rb +238 -0
  253. data/lib/logstash/sized_queue.rb +8 -0
  254. data/lib/logstash/test.rb +183 -0
  255. data/lib/logstash/threadwatchdog.rb +37 -0
  256. data/lib/logstash/time_addon.rb +33 -0
  257. data/lib/logstash/util.rb +106 -0
  258. data/lib/logstash/util/buftok.rb +139 -0
  259. data/lib/logstash/util/charset.rb +39 -0
  260. data/lib/logstash/util/fieldreference.rb +50 -0
  261. data/lib/logstash/util/password.rb +25 -0
  262. data/lib/logstash/util/prctl.rb +11 -0
  263. data/lib/logstash/util/relp.rb +326 -0
  264. data/lib/logstash/util/require-helper.rb +18 -0
  265. data/lib/logstash/util/socket_peer.rb +7 -0
  266. data/lib/logstash/util/zenoss.rb +566 -0
  267. data/lib/logstash/util/zeromq.rb +47 -0
  268. data/lib/logstash/version.rb +6 -0
  269. data/locales/en.yml +170 -0
  270. data/logstash-event.gemspec +29 -0
  271. data/logstash.gemspec +128 -0
  272. data/patterns/firewalls +60 -0
  273. data/patterns/grok-patterns +91 -0
  274. data/patterns/haproxy +37 -0
  275. data/patterns/java +3 -0
  276. data/patterns/linux-syslog +14 -0
  277. data/patterns/mcollective +1 -0
  278. data/patterns/mcollective-patterns +4 -0
  279. data/patterns/nagios +108 -0
  280. data/patterns/postgresql +3 -0
  281. data/patterns/redis +3 -0
  282. data/patterns/ruby +2 -0
  283. data/pkg/build.sh +135 -0
  284. data/pkg/centos/after-install.sh +1 -0
  285. data/pkg/centos/before-install.sh +10 -0
  286. data/pkg/centos/before-remove.sh +11 -0
  287. data/pkg/centos/sysconfig +15 -0
  288. data/pkg/debian/after-install.sh +5 -0
  289. data/pkg/debian/before-install.sh +13 -0
  290. data/pkg/debian/before-remove.sh +13 -0
  291. data/pkg/debian/build.sh +34 -0
  292. data/pkg/debian/debian/README +6 -0
  293. data/pkg/debian/debian/changelog +17 -0
  294. data/pkg/debian/debian/compat +1 -0
  295. data/pkg/debian/debian/control +16 -0
  296. data/pkg/debian/debian/copyright +27 -0
  297. data/pkg/debian/debian/dirs +19 -0
  298. data/pkg/debian/debian/docs +0 -0
  299. data/pkg/debian/debian/logstash.default +39 -0
  300. data/pkg/debian/debian/logstash.init +201 -0
  301. data/pkg/debian/debian/logstash.install +1 -0
  302. data/pkg/debian/debian/logstash.logrotate +9 -0
  303. data/pkg/debian/debian/logstash.postinst +68 -0
  304. data/pkg/debian/debian/logstash.postrm +23 -0
  305. data/pkg/debian/debian/manpage.1.ex +59 -0
  306. data/pkg/debian/debian/preinst.ex +37 -0
  307. data/pkg/debian/debian/prerm.ex +40 -0
  308. data/pkg/debian/debian/release.conf +5 -0
  309. data/pkg/debian/debian/rules +80 -0
  310. data/pkg/debian/debian/watch.ex +22 -0
  311. data/pkg/logrotate.conf +8 -0
  312. data/pkg/logstash-web.default +41 -0
  313. data/pkg/logstash-web.sysv.debian +201 -0
  314. data/pkg/logstash-web.upstart.ubuntu +18 -0
  315. data/pkg/logstash.default +45 -0
  316. data/pkg/logstash.sysv.debian +202 -0
  317. data/pkg/logstash.sysv.redhat +158 -0
  318. data/pkg/logstash.upstart.ubuntu +20 -0
  319. data/pkg/rpm/SOURCES/logstash.conf +26 -0
  320. data/pkg/rpm/SOURCES/logstash.init +80 -0
  321. data/pkg/rpm/SOURCES/logstash.logrotate +8 -0
  322. data/pkg/rpm/SOURCES/logstash.sysconfig +3 -0
  323. data/pkg/rpm/SOURCES/logstash.wrapper +105 -0
  324. data/pkg/rpm/SPECS/logstash.spec +180 -0
  325. data/pkg/rpm/readme.md +4 -0
  326. data/pkg/ubuntu/after-install.sh +7 -0
  327. data/pkg/ubuntu/before-install.sh +12 -0
  328. data/pkg/ubuntu/before-remove.sh +13 -0
  329. data/pull_release_note.rb +25 -0
  330. data/require-analyze.rb +22 -0
  331. data/spec/README.md +14 -0
  332. data/spec/codecs/edn.rb +40 -0
  333. data/spec/codecs/edn_lines.rb +53 -0
  334. data/spec/codecs/graphite.rb +96 -0
  335. data/spec/codecs/json.rb +57 -0
  336. data/spec/codecs/json_lines.rb +51 -0
  337. data/spec/codecs/json_spooler.rb +43 -0
  338. data/spec/codecs/msgpack.rb +39 -0
  339. data/spec/codecs/multiline.rb +60 -0
  340. data/spec/codecs/oldlogstashjson.rb +55 -0
  341. data/spec/codecs/plain.rb +35 -0
  342. data/spec/codecs/spool.rb +35 -0
  343. data/spec/conditionals/test.rb +323 -0
  344. data/spec/config.rb +31 -0
  345. data/spec/event.rb +165 -0
  346. data/spec/examples/fail2ban.rb +28 -0
  347. data/spec/examples/graphite-input.rb +41 -0
  348. data/spec/examples/mysql-slow-query.rb +70 -0
  349. data/spec/examples/parse-apache-logs.rb +66 -0
  350. data/spec/examples/parse-haproxy-logs.rb +115 -0
  351. data/spec/examples/syslog.rb +48 -0
  352. data/spec/filters/alter.rb +96 -0
  353. data/spec/filters/anonymize.rb +189 -0
  354. data/spec/filters/checksum.rb +41 -0
  355. data/spec/filters/clone.rb +67 -0
  356. data/spec/filters/collate.rb +122 -0
  357. data/spec/filters/csv.rb +174 -0
  358. data/spec/filters/date.rb +285 -0
  359. data/spec/filters/date_performance.rb +31 -0
  360. data/spec/filters/dns.rb +159 -0
  361. data/spec/filters/drop.rb +19 -0
  362. data/spec/filters/elapsed.rb +294 -0
  363. data/spec/filters/environment.rb +43 -0
  364. data/spec/filters/geoip.rb +62 -0
  365. data/spec/filters/grep.rb +342 -0
  366. data/spec/filters/grok.rb +473 -0
  367. data/spec/filters/grok/timeout2.rb +56 -0
  368. data/spec/filters/grok/timeouts.rb +39 -0
  369. data/spec/filters/i18n.rb +25 -0
  370. data/spec/filters/json.rb +72 -0
  371. data/spec/filters/json_encode.rb +37 -0
  372. data/spec/filters/kv.rb +403 -0
  373. data/spec/filters/metrics.rb +212 -0
  374. data/spec/filters/multiline.rb +119 -0
  375. data/spec/filters/mutate.rb +180 -0
  376. data/spec/filters/noop.rb +221 -0
  377. data/spec/filters/prune.rb +441 -0
  378. data/spec/filters/punct.rb +18 -0
  379. data/spec/filters/railsparallelrequest.rb +112 -0
  380. data/spec/filters/range.rb +169 -0
  381. data/spec/filters/split.rb +58 -0
  382. data/spec/filters/translate.rb +70 -0
  383. data/spec/filters/unique.rb +25 -0
  384. data/spec/filters/useragent.rb +42 -0
  385. data/spec/filters/xml.rb +157 -0
  386. data/spec/inputs/file.rb +107 -0
  387. data/spec/inputs/gelf.rb +52 -0
  388. data/spec/inputs/generator.rb +30 -0
  389. data/spec/inputs/imap.rb +60 -0
  390. data/spec/inputs/redis.rb +63 -0
  391. data/spec/inputs/relp.rb +70 -0
  392. data/spec/inputs/tcp.rb +101 -0
  393. data/spec/jar.rb +21 -0
  394. data/spec/outputs/csv.rb +266 -0
  395. data/spec/outputs/elasticsearch.rb +161 -0
  396. data/spec/outputs/elasticsearch_http.rb +240 -0
  397. data/spec/outputs/email.rb +173 -0
  398. data/spec/outputs/file.rb +82 -0
  399. data/spec/outputs/graphite.rb +236 -0
  400. data/spec/outputs/redis.rb +127 -0
  401. data/spec/speed.rb +20 -0
  402. data/spec/sqlite-test.rb +81 -0
  403. data/spec/support/LOGSTASH-733.rb +21 -0
  404. data/spec/support/LOGSTASH-820.rb +25 -0
  405. data/spec/support/akamai-grok.rb +26 -0
  406. data/spec/support/date-http.rb +17 -0
  407. data/spec/support/postwait1.rb +26 -0
  408. data/spec/support/pull375.rb +21 -0
  409. data/spec/test_utils.rb +125 -0
  410. data/spec/util/fieldeval_spec.rb +44 -0
  411. data/test/jenkins/config.xml.erb +74 -0
  412. data/test/jenkins/create-jobs.rb +23 -0
  413. data/test/jenkins/generatorjob.config.xml +66 -0
  414. data/tools/Gemfile +14 -0
  415. data/tools/Gemfile.jruby-1.9.lock +322 -0
  416. data/tools/Gemfile.rbx-2.1.lock +516 -0
  417. data/tools/Gemfile.ruby-1.9.1.lock +310 -0
  418. data/tools/Gemfile.ruby-2.0.0.lock +310 -0
  419. metadata +629 -0
@@ -0,0 +1,80 @@
1
+ #!/bin/bash
2
+ # chkconfig: - 80 15
3
+ ### BEGIN INIT INFO
4
+ # Provides: logstash
5
+ # Required-Start: $all
6
+ # Required-Stop: $all
7
+ # Default-Start:
8
+ # Default-Stop: 0 1 6
9
+ # Short-Description: Starts logstash
10
+ # Description: Logstash agent
11
+ ### END INIT INFO
12
+
13
+ # Source function library.
14
+ . /etc/rc.d/init.d/functions
15
+
16
+ NAME=@@@NAME@@@
17
+
18
+ [ -f /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME
19
+
20
+ LOGSTASH_USER=${NAME}
21
+ DAEMON="@@@DAEMON@@@/${NAME}"
22
+ SERVICE=agent
23
+
24
+ PID_FILE=${PIDFILE:-@@@PIDDIR@@@/${NAME}.pid}
25
+ LOCK_FILE=${LOCKFILE:-@@@LOCKFILE@@@}
26
+ LOG_FILE=${LOGFILE:-@@@LOGDIR@@@/${NAME}.log}
27
+
28
+ LOGSTASH_PATH_CONF=${LOGSTASH_PATH_CONF:-@@@CONFDIR@@@}
29
+ LOGSTASH_LOGLEVEL=${LOGSTASH_LOGLEVEL:-"warn"}
30
+
31
+ DAEMON_OPTS="\
32
+ -P ${PID_FILE} \
33
+ -l ${LOG_FILE} \
34
+ -f ${LOGSTASH_PATH_CONF} \
35
+ -v $LOGSTASH_LOGLEVEL \
36
+ "
37
+
38
+ start() {
39
+ echo -n $"Starting ${NAME}: "
40
+ export JAVA_OPTS="$JAVA_OPTS $LOGSTASH_JAVA_OPTS"
41
+ daemon --pidfile=${PID_FILE} --user $LOGSTASH_USER $DAEMON $SERVICE $DAEMON_OPTS
42
+ RETVAL=$?
43
+ echo
44
+ [ $RETVAL -eq 0 ] && touch $LOCK_FILE
45
+ return $RETVAL
46
+ }
47
+
48
+ stop() {
49
+
50
+ echo -n $"Stopping ${NAME}: "
51
+ killproc -p ${PID_FILE} -d 10 $DAEMON
52
+ RETVAL=$?
53
+ echo
54
+ [ $RETVAL = 0 ] && rm -f ${LOCK_FILE} ${PID_FILE}
55
+ return $RETVAL
56
+ }
57
+
58
+ case "$1" in
59
+ start)
60
+ start
61
+ ;;
62
+ stop)
63
+ stop
64
+ ;;
65
+ status)
66
+ status -p ${PID_FILE} $DAEMON
67
+ RETVAL=$?
68
+ ;;
69
+ restart|force-reload)
70
+ stop
71
+ start
72
+ ;;
73
+ *)
74
+ N=/etc/init.d/${NAME}
75
+ echo "Usage: $N {start|stop|restart|force-reload}" >&2
76
+ RETVAL=2
77
+ ;;
78
+ esac
79
+
80
+ exit $RETVAL
@@ -0,0 +1,8 @@
1
+ /var/log/logstash/*.log {
2
+ daily
3
+ rotate 7
4
+ copytruncate
5
+ compress
6
+ missingok
7
+ notifempty
8
+ }
@@ -0,0 +1,3 @@
1
+ #LOGSTASH_LOGFILE=@@@LOGDIR@@@/@@@NAME@@@.log
2
+ #LOGSTASH_PATH_CONF=@@@CONFDIR@@@
3
+ #LOGSTASH_JAVA_OPTS="-Djava.io.tmpdir=/opt/@@@NAME@@@/tmp"
@@ -0,0 +1,105 @@
1
+ #!/bin/sh
2
+ SCRIPT=$0
3
+
4
+ if [ -x "$JAVA_HOME/bin/java" ]; then
5
+ JAVA=$JAVA_HOME/bin/java
6
+ else
7
+ JAVA=`which java`
8
+ fi
9
+
10
+ LOGSTASH_JAR="@@@JARPATH@@@/@@@NAME@@@.jar"
11
+ if [ ! -f $LOGSTASH_JAR ]
12
+ then
13
+ echo "jar file is not found."
14
+ exit 99
15
+ fi
16
+
17
+ function usage() {
18
+ echo "Usage: ${SCRIPT} SERVICE OPTIONS"
19
+ echo " SERVICE: agent, web"
20
+ echo " OPTIONS:"
21
+ echo -e " -f, --config CONFIGFILE (required)\tLoad the logstash config from a specific file or directory."
22
+ echo -e " \t\tIf a direcory is given instead of a file, all files in that directory will be concatonated in lexicographical order and then parsed as a single config file."
23
+ echo -e " -P, --pidfile PIDFILE\t\tPID file path."
24
+ echo -e " -l, --logfile LOGFILE\t\tLogfile path."
25
+ echo -e " -v, --verbose [info, debug]\t\tEnables more verbose logging"
26
+ }
27
+
28
+ function run_service() {
29
+ service=$1
30
+ config=$2
31
+ pidfile=$3
32
+ logfile=$4
33
+ verbose=$5
34
+
35
+ if [ "x$logfile" == "x" ]; then
36
+ exec "$JAVA" $JAVA_OPTS -jar $LOGSTASH_JAR $service $verbose -f "$config" &
37
+ rs=$?
38
+ else
39
+ exec "$JAVA" $JAVA_OPTS -jar $LOGSTASH_JAR $service $verbose -f "$config" -l "$logfile" 2>&1 >> $logfile &
40
+ rs=$?
41
+ [ $rs -eq 0 -a "x$pidfile" != "x" ] && printf '%d' $! > "$pidfile"
42
+ fi
43
+
44
+ return $rs
45
+ }
46
+
47
+ service=$1; shift
48
+
49
+ if [ "${service}" != "agent" -a "${service}" != "web" ]
50
+ then
51
+ echo "ERROR: no such service \`${service}'. Available services are: agent, web"
52
+ usage
53
+ exit 99
54
+ fi
55
+
56
+ while test $# -gt 0
57
+ do
58
+ case "$1" in
59
+ -V | --version )
60
+ "$JAVA" -jar $LOGSTASH_JAR --version
61
+ exit 0
62
+ ;;
63
+ -f | --config )
64
+ config="$2"
65
+ if [ ! -f "$config" -a ! -d "$config" ]; then
66
+ echo "ERROR: config file or directory \`$config' does not exist."
67
+ usage
68
+ exit 1
69
+ fi
70
+ shift 2
71
+ ;;
72
+ -P | --pidfile )
73
+ pidfile="$2"
74
+ shift 2
75
+ ;;
76
+ -l | --logfile )
77
+ logfile="$2"
78
+ shift 2
79
+ ;;
80
+ -v | --verbose )
81
+ if [ "$2" == "debug" ]; then
82
+ verbose="-vv"
83
+ elif [ "$2" == "info" ]; then
84
+ verbose="-v"
85
+ fi
86
+ shift 2
87
+ ;;
88
+ -h | --help )
89
+ usage
90
+ exit 0
91
+ ;;
92
+ * )
93
+ break
94
+ esac
95
+ done
96
+
97
+ if [ "x$config" == "x" ]; then
98
+ echo "ERROR: config file is required."
99
+ usage
100
+ exit 1
101
+ fi
102
+
103
+ run_service "$service" "$config" "$pidfile" "$logfile" "$verbose"
104
+
105
+ exit $?
@@ -0,0 +1,180 @@
1
+ %define debug_package %{nil}_bindir}
2
+ %define base_install_dir %{_javadir}{%name}
3
+
4
+ %global bindir %{_bindir}
5
+ %global confdir %{_sysconfdir}/%{name}
6
+ %global jarpath %{_javadir}
7
+ %global lockfile %{_localstatedir}/lock/subsys/%{name}
8
+ %global logdir %{_localstatedir}/log/%{name}
9
+ %global piddir %{_localstatedir}/run/%{name}
10
+ %global sysconfigdir %{_sysconfdir}/sysconfig
11
+
12
+ Name: logstash
13
+ Version: 1.2.2
14
+ Release: 2%{?dist}
15
+ Summary: A tool for managing events and logs
16
+
17
+ Group: System Environment/Daemons
18
+ License: ASL 2.0
19
+ URL: http://logstash.net
20
+ Source0: https://logstash.objects.dreamhost.com/release/%{name}-%{version}-flatjar.jar
21
+ Source1: logstash.wrapper
22
+ Source2: logstash.logrotate
23
+ Source3: logstash.init
24
+ Source4: logstash.sysconfig
25
+ Source5: logstash.conf
26
+ BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
27
+ BuildArch: x86_64 i386 i686
28
+
29
+ Requires: java
30
+ Requires: jpackage-utils
31
+
32
+ Requires(post): chkconfig initscripts
33
+ Requires(pre): chkconfig initscripts
34
+ Requires(pre): shadow-utils
35
+
36
+ %description
37
+ A tool for managing events and logs.
38
+
39
+ %prep
40
+ %build
41
+
42
+ %install
43
+ rm -rf $RPM_BUILD_ROOT
44
+
45
+ # JAR file
46
+ %{__mkdir} -p %{buildroot}%{_javadir}
47
+ %{__install} -p -m 644 %{SOURCE0} %{buildroot}%{jarpath}/%{name}.jar
48
+
49
+ # Config
50
+ %{__mkdir} -p %{buildroot}%{confdir}
51
+ %{__install} -m 644 %{SOURCE5} %{buildroot}%{confdir}/%{name}.conf
52
+
53
+ # Wrapper script
54
+ %{__mkdir} -p %{buildroot}%{_bindir}
55
+ %{__install} -m 755 %{SOURCE1} %{buildroot}%{bindir}/%{name}
56
+
57
+ %{__sed} -i \
58
+ -e "s|@@@NAME@@@|%{name}|g" \
59
+ -e "s|@@@JARPATH@@@|%{jarpath}|g" \
60
+ %{buildroot}%{bindir}/%{name}
61
+
62
+ # Logs
63
+ %{__mkdir} -p %{buildroot}%{logdir}
64
+ %{__install} -D -m 644 %{SOURCE2} %{buildroot}%{_sysconfdir}/logrotate.d/%{name}
65
+
66
+ # Misc
67
+ %{__mkdir} -p %{buildroot}%{piddir}
68
+
69
+ # sysconfig and init
70
+ %{__mkdir} -p %{buildroot}%{_initddir}
71
+ %{__mkdir} -p %{buildroot}%{_sysconfdir}/sysconfig
72
+ %{__install} -m 755 %{SOURCE3} %{buildroot}%{_initddir}/%{name}
73
+ %{__install} -m 644 %{SOURCE4} %{buildroot}%{sysconfigdir}/%{name}
74
+
75
+ %{__sed} -i \
76
+ -e "s|@@@NAME@@@|%{name}|g" \
77
+ -e "s|@@@DAEMON@@@|%{bindir}|g" \
78
+ -e "s|@@@CONFDIR@@@|%{confdir}|g" \
79
+ -e "s|@@@LOCKFILE@@@|%{lockfile}|g" \
80
+ -e "s|@@@LOGDIR@@@|%{logdir}|g" \
81
+ -e "s|@@@PIDDIR@@@|%{piddir}|g" \
82
+ %{buildroot}%{_initddir}/%{name}
83
+
84
+ %{__sed} -i \
85
+ -e "s|@@@NAME@@@|%{name}|g" \
86
+ -e "s|@@@CONFDIR@@@|%{confdir}|g" \
87
+ -e "s|@@@LOGDIR@@@|%{logdir}|g" \
88
+ -e "s|@@@PLUGINDIR@@@|%{_datadir}|g" \
89
+ %{buildroot}%{sysconfigdir}/%{name}
90
+
91
+ %pre
92
+ # create logstash group
93
+ if ! getent group logstash >/dev/null; then
94
+ groupadd -r logstash
95
+ fi
96
+
97
+ # create logstash user
98
+ if ! getent passwd logstash >/dev/null; then
99
+ useradd -r -g logstash -d %{_javadir}/%{name} \
100
+ -s /sbin/nologin -c "You know, for search" logstash
101
+ fi
102
+
103
+ %post
104
+ /sbin/chkconfig --add logstash
105
+
106
+ %preun
107
+ if [ $1 -eq 0 ]; then
108
+ /sbin/service logstash stop >/dev/null 2>&1
109
+ /sbin/chkconfig --del logstash
110
+ fi
111
+
112
+ %clean
113
+ rm -rf $RPM_BUILD_ROOT
114
+
115
+ %files
116
+ %defattr(-,root,root,-)
117
+ # JAR file
118
+ %{_javadir}/%{name}.jar
119
+
120
+ # Config
121
+ %config(noreplace) %{confdir}/
122
+
123
+ # Wrapper script
124
+ %{bindir}/*
125
+
126
+
127
+ # Logrotate
128
+ %config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
129
+
130
+ # Sysconfig and init
131
+ %{_initddir}/%{name}
132
+ %config(noreplace) %{sysconfigdir}/*
133
+
134
+ %defattr(-,%{name},%{name},-)
135
+ %dir %{logdir}/
136
+ %dir %{piddir}/
137
+
138
+ %changelog
139
+ * Sun Oct 27 2013 <sbagmeijer@ulyaoth.asia> - 1.2.2-2
140
+ - Fixed the bogus date warning
141
+
142
+ * Wed Oct 23 2013 <sjir@basefarm.no> - 1.2.2-1
143
+ - Update logstash version to 1.2.2-1
144
+
145
+ * Fri Sep 27 2013 <sbagmeijer@ulyaoth.asia> - 1.2.1-1
146
+ - Building for Fedora 18, 19 and 20-alpha.
147
+ - Added Source5 for logstash.conf
148
+ - Fixed the BuildArch so it can be build for i386
149
+
150
+ * Mon Sep 16 2013 <sjir@basefarm.se> - 1.2.1
151
+ - Updated version to the new 1.2.1
152
+ - Removed everything related to plugins as it no longer works.
153
+
154
+ * Wed Sep 04 2013 <sjir@basefarm.se> - 1.2.0
155
+ - Updated version to the new 1.2.0.
156
+ - Fixed a problem with the init.d script not working correctly.
157
+
158
+ * Fri Jun 14 2013 <sjir@basefarm.se> - 1.1.13-1
159
+ - Updated version to the new 1.1.13-1 and fixed some minor issues with directory structure.
160
+
161
+ * Mon May 6 2013 <sjir@basefarm.se> - 1.1.10-3
162
+ - Changed from logstash flatjar to the monolith as flatjar is not working correctly yet.
163
+
164
+ * Fri Apr 19 2013 <sjir@basefarm.se> - 1.1.10-2
165
+ - Fixed a bug
166
+
167
+ * Fri Apr 19 2013 <sjir@basefarm.se> - 1.1.10-1
168
+ - Added fixes to support RHEL6
169
+ - Update logstash version to 1.1.10
170
+
171
+ * Sun Mar 17 2013 Richard Pijnenburg <richard@ispavailability.com> - 1.1.9-2
172
+ - Update init script
173
+ - Create patterns dir in correct place
174
+
175
+ * Fri Feb 1 2013 Richard Pijnenburg <richard@ispavailability.com> - 1.1.9-1
176
+ - Update to latest stable release.
177
+ - New init script
178
+
179
+ * Fri May 4 2012 Maksim Horbul <max@gorbul.net> - 1.1.0-1
180
+ - Initial package
@@ -0,0 +1,4 @@
1
+ # RPM build
2
+
3
+ spectool -g SPECS/logstash.spec
4
+ rpmbuild -bb SPECS/logstash.spec
@@ -0,0 +1,7 @@
1
+ #!/bin/sh
2
+
3
+ chown -R logstash:logstash /opt/logstash
4
+ chown logstash /var/log/logstash
5
+ chown logstash:logstash /var/lib/logstash
6
+
7
+ ln -sf /opt/logstash/logstash.jar /var/lib/logstash/logstash.jar
@@ -0,0 +1,12 @@
1
+ #!/bin/sh
2
+
3
+ # create logstash group
4
+ if ! getent group logstash >/dev/null; then
5
+ groupadd -r logstash
6
+ fi
7
+
8
+ # create logstash user
9
+ if ! getent passwd logstash >/dev/null; then
10
+ useradd -M -r -g logstash -d /var/lib/logstash \
11
+ -s /sbin/nologin -c "LogStash Service User" logstash
12
+ fi
@@ -0,0 +1,13 @@
1
+ #!/bin/sh
2
+
3
+ if [ $1 == "remove" ]; then
4
+ stop logstash >/dev/null 2>&1 || true
5
+
6
+ if getent passwd logstash >/dev/null ; then
7
+ userdel logstash
8
+ fi
9
+
10
+ if getent group logstash >/dev/null ; then
11
+ groupdel logstash
12
+ fi
13
+ fi
@@ -0,0 +1,25 @@
1
+ require "octokit"
2
+
3
+
4
+ @repository= "logstash/logstash"
5
+ @releaseNote= "releaseNote.html"
6
+
7
+ #Last release == last tag
8
+ lastReleaseSha = Octokit.tags(@repository).first.commit.sha
9
+
10
+ currentReleaseSha ="HEAD"
11
+
12
+ #Collect PR Merge in a file
13
+ File.open(@releaseNote, "a") do |f|
14
+ f.puts "<h2>Merged pull request</h2>"
15
+ f.puts "<ul>"
16
+ Octokit.compare(@repository, lastReleaseSha, currentReleaseSha).commits.each do |commit|
17
+ if commit.commit.message.start_with?("Merge pull")
18
+ scan_re = Regexp.new(/^Merge pull request #(\d+) from ([^\/]+)\/.*\n\n(.*)/)
19
+ commit.commit.message.scan(scan_re) do |pullNumber, user, summary|
20
+ f.puts "<li><a href='https://github.com/logstash/logstash/pull/#{pullNumber}'>Pull ##{pullNumber}<a> by #{user}: #{summary}</li>"
21
+ end
22
+ end
23
+ end
24
+ f.puts "</ul>"
25
+ end