logstash-lib 1.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (419) hide show
  1. data/.gitignore +24 -0
  2. data/.tailor +8 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG +1185 -0
  5. data/CONTRIBUTING.md +61 -0
  6. data/CONTRIBUTORS +79 -0
  7. data/LICENSE +14 -0
  8. data/Makefile +460 -0
  9. data/README.md +120 -0
  10. data/STYLE.md +96 -0
  11. data/bin/logstash +37 -0
  12. data/bin/logstash-test +4 -0
  13. data/bin/logstash-web +4 -0
  14. data/bin/logstash.lib.sh +78 -0
  15. data/bot/check_pull_changelog.rb +89 -0
  16. data/docs/configuration.md +260 -0
  17. data/docs/docgen.rb +242 -0
  18. data/docs/extending/example-add-a-new-filter.md +121 -0
  19. data/docs/extending/index.md +91 -0
  20. data/docs/flags.md +43 -0
  21. data/docs/generate_index.rb +28 -0
  22. data/docs/index.html.erb +56 -0
  23. data/docs/learn.md +46 -0
  24. data/docs/life-of-an-event.md +109 -0
  25. data/docs/logging-tool-comparisons.md +60 -0
  26. data/docs/plugin-doc.html.erb +91 -0
  27. data/docs/plugin-milestones.md +41 -0
  28. data/docs/plugin-synopsis.html.erb +24 -0
  29. data/docs/release-engineering.md +46 -0
  30. data/docs/release-test-results.md +14 -0
  31. data/docs/repositories.md +35 -0
  32. data/docs/tutorials/10-minute-walkthrough/apache-elasticsearch.conf +35 -0
  33. data/docs/tutorials/10-minute-walkthrough/apache-parse.conf +33 -0
  34. data/docs/tutorials/10-minute-walkthrough/apache_log.1 +1 -0
  35. data/docs/tutorials/10-minute-walkthrough/apache_log.2.bz2 +0 -0
  36. data/docs/tutorials/10-minute-walkthrough/hello-search.conf +25 -0
  37. data/docs/tutorials/10-minute-walkthrough/hello.conf +16 -0
  38. data/docs/tutorials/10-minute-walkthrough/index.md +124 -0
  39. data/docs/tutorials/10-minute-walkthrough/step-5-output.txt +17 -0
  40. data/docs/tutorials/getting-started-centralized-overview-diagram.png +0 -0
  41. data/docs/tutorials/getting-started-centralized-overview-diagram.xml +1 -0
  42. data/docs/tutorials/getting-started-centralized.md +217 -0
  43. data/docs/tutorials/getting-started-simple.md +200 -0
  44. data/docs/tutorials/just-enough-rabbitmq-for-logstash.md +201 -0
  45. data/docs/tutorials/media/frontend-response-codes.png +0 -0
  46. data/docs/tutorials/metrics-from-logs.md +84 -0
  47. data/docs/tutorials/zeromq.md +118 -0
  48. data/extract_services.rb +29 -0
  49. data/gembag.rb +64 -0
  50. data/lib/logstash-event.rb +2 -0
  51. data/lib/logstash.rb +4 -0
  52. data/lib/logstash/JRUBY-6970-openssl.rb +22 -0
  53. data/lib/logstash/JRUBY-6970.rb +102 -0
  54. data/lib/logstash/agent.rb +305 -0
  55. data/lib/logstash/certs/cacert.pem +3895 -0
  56. data/lib/logstash/codecs/base.rb +49 -0
  57. data/lib/logstash/codecs/compress_spooler.rb +50 -0
  58. data/lib/logstash/codecs/dots.rb +18 -0
  59. data/lib/logstash/codecs/edn.rb +28 -0
  60. data/lib/logstash/codecs/edn_lines.rb +36 -0
  61. data/lib/logstash/codecs/fluent.rb +55 -0
  62. data/lib/logstash/codecs/graphite.rb +114 -0
  63. data/lib/logstash/codecs/json.rb +41 -0
  64. data/lib/logstash/codecs/json_lines.rb +52 -0
  65. data/lib/logstash/codecs/json_spooler.rb +22 -0
  66. data/lib/logstash/codecs/line.rb +58 -0
  67. data/lib/logstash/codecs/msgpack.rb +43 -0
  68. data/lib/logstash/codecs/multiline.rb +189 -0
  69. data/lib/logstash/codecs/netflow.rb +342 -0
  70. data/lib/logstash/codecs/netflow/util.rb +212 -0
  71. data/lib/logstash/codecs/noop.rb +19 -0
  72. data/lib/logstash/codecs/oldlogstashjson.rb +56 -0
  73. data/lib/logstash/codecs/plain.rb +48 -0
  74. data/lib/logstash/codecs/rubydebug.rb +22 -0
  75. data/lib/logstash/codecs/spool.rb +38 -0
  76. data/lib/logstash/config/Makefile +4 -0
  77. data/lib/logstash/config/config_ast.rb +380 -0
  78. data/lib/logstash/config/file.rb +39 -0
  79. data/lib/logstash/config/grammar.rb +3504 -0
  80. data/lib/logstash/config/grammar.treetop +241 -0
  81. data/lib/logstash/config/mixin.rb +464 -0
  82. data/lib/logstash/config/registry.rb +13 -0
  83. data/lib/logstash/config/test.conf +18 -0
  84. data/lib/logstash/errors.rb +10 -0
  85. data/lib/logstash/event.rb +262 -0
  86. data/lib/logstash/filters/advisor.rb +178 -0
  87. data/lib/logstash/filters/alter.rb +173 -0
  88. data/lib/logstash/filters/anonymize.rb +93 -0
  89. data/lib/logstash/filters/base.rb +190 -0
  90. data/lib/logstash/filters/checksum.rb +50 -0
  91. data/lib/logstash/filters/cidr.rb +76 -0
  92. data/lib/logstash/filters/cipher.rb +145 -0
  93. data/lib/logstash/filters/clone.rb +35 -0
  94. data/lib/logstash/filters/collate.rb +114 -0
  95. data/lib/logstash/filters/csv.rb +94 -0
  96. data/lib/logstash/filters/date.rb +244 -0
  97. data/lib/logstash/filters/dns.rb +201 -0
  98. data/lib/logstash/filters/drop.rb +32 -0
  99. data/lib/logstash/filters/elapsed.rb +256 -0
  100. data/lib/logstash/filters/elasticsearch.rb +73 -0
  101. data/lib/logstash/filters/environment.rb +27 -0
  102. data/lib/logstash/filters/extractnumbers.rb +84 -0
  103. data/lib/logstash/filters/gelfify.rb +52 -0
  104. data/lib/logstash/filters/geoip.rb +145 -0
  105. data/lib/logstash/filters/grep.rb +153 -0
  106. data/lib/logstash/filters/grok.rb +425 -0
  107. data/lib/logstash/filters/grokdiscovery.rb +75 -0
  108. data/lib/logstash/filters/i18n.rb +51 -0
  109. data/lib/logstash/filters/json.rb +90 -0
  110. data/lib/logstash/filters/json_encode.rb +52 -0
  111. data/lib/logstash/filters/kv.rb +232 -0
  112. data/lib/logstash/filters/metaevent.rb +68 -0
  113. data/lib/logstash/filters/metrics.rb +237 -0
  114. data/lib/logstash/filters/multiline.rb +241 -0
  115. data/lib/logstash/filters/mutate.rb +399 -0
  116. data/lib/logstash/filters/noop.rb +21 -0
  117. data/lib/logstash/filters/prune.rb +149 -0
  118. data/lib/logstash/filters/punct.rb +32 -0
  119. data/lib/logstash/filters/railsparallelrequest.rb +86 -0
  120. data/lib/logstash/filters/range.rb +142 -0
  121. data/lib/logstash/filters/ruby.rb +42 -0
  122. data/lib/logstash/filters/sleep.rb +111 -0
  123. data/lib/logstash/filters/split.rb +64 -0
  124. data/lib/logstash/filters/sumnumbers.rb +73 -0
  125. data/lib/logstash/filters/syslog_pri.rb +107 -0
  126. data/lib/logstash/filters/translate.rb +121 -0
  127. data/lib/logstash/filters/unique.rb +29 -0
  128. data/lib/logstash/filters/urldecode.rb +57 -0
  129. data/lib/logstash/filters/useragent.rb +112 -0
  130. data/lib/logstash/filters/uuid.rb +58 -0
  131. data/lib/logstash/filters/xml.rb +139 -0
  132. data/lib/logstash/filters/zeromq.rb +123 -0
  133. data/lib/logstash/filterworker.rb +122 -0
  134. data/lib/logstash/inputs/base.rb +125 -0
  135. data/lib/logstash/inputs/collectd.rb +306 -0
  136. data/lib/logstash/inputs/drupal_dblog.rb +323 -0
  137. data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb +66 -0
  138. data/lib/logstash/inputs/elasticsearch.rb +140 -0
  139. data/lib/logstash/inputs/eventlog.rb +129 -0
  140. data/lib/logstash/inputs/eventlog/racob_fix.rb +44 -0
  141. data/lib/logstash/inputs/exec.rb +69 -0
  142. data/lib/logstash/inputs/file.rb +146 -0
  143. data/lib/logstash/inputs/ganglia.rb +127 -0
  144. data/lib/logstash/inputs/ganglia/gmondpacket.rb +146 -0
  145. data/lib/logstash/inputs/ganglia/xdr.rb +327 -0
  146. data/lib/logstash/inputs/gelf.rb +138 -0
  147. data/lib/logstash/inputs/gemfire.rb +222 -0
  148. data/lib/logstash/inputs/generator.rb +97 -0
  149. data/lib/logstash/inputs/graphite.rb +41 -0
  150. data/lib/logstash/inputs/heroku.rb +51 -0
  151. data/lib/logstash/inputs/imap.rb +136 -0
  152. data/lib/logstash/inputs/irc.rb +84 -0
  153. data/lib/logstash/inputs/log4j.rb +136 -0
  154. data/lib/logstash/inputs/lumberjack.rb +53 -0
  155. data/lib/logstash/inputs/pipe.rb +57 -0
  156. data/lib/logstash/inputs/rabbitmq.rb +126 -0
  157. data/lib/logstash/inputs/rabbitmq/bunny.rb +118 -0
  158. data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb +1 -0
  159. data/lib/logstash/inputs/rabbitmq/march_hare.rb +129 -0
  160. data/lib/logstash/inputs/redis.rb +263 -0
  161. data/lib/logstash/inputs/relp.rb +106 -0
  162. data/lib/logstash/inputs/s3.rb +279 -0
  163. data/lib/logstash/inputs/snmptrap.rb +87 -0
  164. data/lib/logstash/inputs/sqlite.rb +185 -0
  165. data/lib/logstash/inputs/sqs.rb +172 -0
  166. data/lib/logstash/inputs/stdin.rb +46 -0
  167. data/lib/logstash/inputs/stomp.rb +84 -0
  168. data/lib/logstash/inputs/syslog.rb +237 -0
  169. data/lib/logstash/inputs/tcp.rb +231 -0
  170. data/lib/logstash/inputs/threadable.rb +18 -0
  171. data/lib/logstash/inputs/twitter.rb +82 -0
  172. data/lib/logstash/inputs/udp.rb +81 -0
  173. data/lib/logstash/inputs/unix.rb +163 -0
  174. data/lib/logstash/inputs/varnishlog.rb +48 -0
  175. data/lib/logstash/inputs/websocket.rb +50 -0
  176. data/lib/logstash/inputs/wmi.rb +72 -0
  177. data/lib/logstash/inputs/xmpp.rb +81 -0
  178. data/lib/logstash/inputs/zenoss.rb +143 -0
  179. data/lib/logstash/inputs/zeromq.rb +165 -0
  180. data/lib/logstash/kibana.rb +113 -0
  181. data/lib/logstash/loadlibs.rb +9 -0
  182. data/lib/logstash/logging.rb +89 -0
  183. data/lib/logstash/monkeypatches-for-bugs.rb +2 -0
  184. data/lib/logstash/monkeypatches-for-debugging.rb +47 -0
  185. data/lib/logstash/monkeypatches-for-performance.rb +66 -0
  186. data/lib/logstash/multiqueue.rb +53 -0
  187. data/lib/logstash/namespace.rb +16 -0
  188. data/lib/logstash/outputs/base.rb +120 -0
  189. data/lib/logstash/outputs/boundary.rb +116 -0
  190. data/lib/logstash/outputs/circonus.rb +78 -0
  191. data/lib/logstash/outputs/cloudwatch.rb +351 -0
  192. data/lib/logstash/outputs/csv.rb +55 -0
  193. data/lib/logstash/outputs/datadog.rb +93 -0
  194. data/lib/logstash/outputs/datadog_metrics.rb +123 -0
  195. data/lib/logstash/outputs/elasticsearch.rb +332 -0
  196. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +44 -0
  197. data/lib/logstash/outputs/elasticsearch_http.rb +256 -0
  198. data/lib/logstash/outputs/elasticsearch_river.rb +214 -0
  199. data/lib/logstash/outputs/email.rb +299 -0
  200. data/lib/logstash/outputs/exec.rb +40 -0
  201. data/lib/logstash/outputs/file.rb +180 -0
  202. data/lib/logstash/outputs/ganglia.rb +75 -0
  203. data/lib/logstash/outputs/gelf.rb +208 -0
  204. data/lib/logstash/outputs/gemfire.rb +103 -0
  205. data/lib/logstash/outputs/google_bigquery.rb +570 -0
  206. data/lib/logstash/outputs/google_cloud_storage.rb +431 -0
  207. data/lib/logstash/outputs/graphite.rb +143 -0
  208. data/lib/logstash/outputs/graphtastic.rb +185 -0
  209. data/lib/logstash/outputs/hipchat.rb +80 -0
  210. data/lib/logstash/outputs/http.rb +142 -0
  211. data/lib/logstash/outputs/irc.rb +80 -0
  212. data/lib/logstash/outputs/jira.rb +109 -0
  213. data/lib/logstash/outputs/juggernaut.rb +105 -0
  214. data/lib/logstash/outputs/librato.rb +146 -0
  215. data/lib/logstash/outputs/loggly.rb +93 -0
  216. data/lib/logstash/outputs/lumberjack.rb +51 -0
  217. data/lib/logstash/outputs/metriccatcher.rb +103 -0
  218. data/lib/logstash/outputs/mongodb.rb +81 -0
  219. data/lib/logstash/outputs/nagios.rb +119 -0
  220. data/lib/logstash/outputs/nagios_nsca.rb +123 -0
  221. data/lib/logstash/outputs/null.rb +18 -0
  222. data/lib/logstash/outputs/opentsdb.rb +101 -0
  223. data/lib/logstash/outputs/pagerduty.rb +79 -0
  224. data/lib/logstash/outputs/pipe.rb +132 -0
  225. data/lib/logstash/outputs/rabbitmq.rb +96 -0
  226. data/lib/logstash/outputs/rabbitmq/bunny.rb +135 -0
  227. data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +1 -0
  228. data/lib/logstash/outputs/rabbitmq/march_hare.rb +143 -0
  229. data/lib/logstash/outputs/redis.rb +245 -0
  230. data/lib/logstash/outputs/riak.rb +152 -0
  231. data/lib/logstash/outputs/riemann.rb +109 -0
  232. data/lib/logstash/outputs/s3.rb +356 -0
  233. data/lib/logstash/outputs/sns.rb +124 -0
  234. data/lib/logstash/outputs/solr_http.rb +78 -0
  235. data/lib/logstash/outputs/sqs.rb +141 -0
  236. data/lib/logstash/outputs/statsd.rb +116 -0
  237. data/lib/logstash/outputs/stdout.rb +53 -0
  238. data/lib/logstash/outputs/stomp.rb +67 -0
  239. data/lib/logstash/outputs/syslog.rb +145 -0
  240. data/lib/logstash/outputs/tcp.rb +145 -0
  241. data/lib/logstash/outputs/udp.rb +38 -0
  242. data/lib/logstash/outputs/websocket.rb +46 -0
  243. data/lib/logstash/outputs/websocket/app.rb +29 -0
  244. data/lib/logstash/outputs/websocket/pubsub.rb +45 -0
  245. data/lib/logstash/outputs/xmpp.rb +78 -0
  246. data/lib/logstash/outputs/zabbix.rb +108 -0
  247. data/lib/logstash/outputs/zeromq.rb +125 -0
  248. data/lib/logstash/pipeline.rb +286 -0
  249. data/lib/logstash/plugin.rb +150 -0
  250. data/lib/logstash/plugin_mixins/aws_config.rb +93 -0
  251. data/lib/logstash/program.rb +15 -0
  252. data/lib/logstash/runner.rb +238 -0
  253. data/lib/logstash/sized_queue.rb +8 -0
  254. data/lib/logstash/test.rb +183 -0
  255. data/lib/logstash/threadwatchdog.rb +37 -0
  256. data/lib/logstash/time_addon.rb +33 -0
  257. data/lib/logstash/util.rb +106 -0
  258. data/lib/logstash/util/buftok.rb +139 -0
  259. data/lib/logstash/util/charset.rb +39 -0
  260. data/lib/logstash/util/fieldreference.rb +50 -0
  261. data/lib/logstash/util/password.rb +25 -0
  262. data/lib/logstash/util/prctl.rb +11 -0
  263. data/lib/logstash/util/relp.rb +326 -0
  264. data/lib/logstash/util/require-helper.rb +18 -0
  265. data/lib/logstash/util/socket_peer.rb +7 -0
  266. data/lib/logstash/util/zenoss.rb +566 -0
  267. data/lib/logstash/util/zeromq.rb +47 -0
  268. data/lib/logstash/version.rb +6 -0
  269. data/locales/en.yml +170 -0
  270. data/logstash-event.gemspec +29 -0
  271. data/logstash.gemspec +128 -0
  272. data/patterns/firewalls +60 -0
  273. data/patterns/grok-patterns +91 -0
  274. data/patterns/haproxy +37 -0
  275. data/patterns/java +3 -0
  276. data/patterns/linux-syslog +14 -0
  277. data/patterns/mcollective +1 -0
  278. data/patterns/mcollective-patterns +4 -0
  279. data/patterns/nagios +108 -0
  280. data/patterns/postgresql +3 -0
  281. data/patterns/redis +3 -0
  282. data/patterns/ruby +2 -0
  283. data/pkg/build.sh +135 -0
  284. data/pkg/centos/after-install.sh +1 -0
  285. data/pkg/centos/before-install.sh +10 -0
  286. data/pkg/centos/before-remove.sh +11 -0
  287. data/pkg/centos/sysconfig +15 -0
  288. data/pkg/debian/after-install.sh +5 -0
  289. data/pkg/debian/before-install.sh +13 -0
  290. data/pkg/debian/before-remove.sh +13 -0
  291. data/pkg/debian/build.sh +34 -0
  292. data/pkg/debian/debian/README +6 -0
  293. data/pkg/debian/debian/changelog +17 -0
  294. data/pkg/debian/debian/compat +1 -0
  295. data/pkg/debian/debian/control +16 -0
  296. data/pkg/debian/debian/copyright +27 -0
  297. data/pkg/debian/debian/dirs +19 -0
  298. data/pkg/debian/debian/docs +0 -0
  299. data/pkg/debian/debian/logstash.default +39 -0
  300. data/pkg/debian/debian/logstash.init +201 -0
  301. data/pkg/debian/debian/logstash.install +1 -0
  302. data/pkg/debian/debian/logstash.logrotate +9 -0
  303. data/pkg/debian/debian/logstash.postinst +68 -0
  304. data/pkg/debian/debian/logstash.postrm +23 -0
  305. data/pkg/debian/debian/manpage.1.ex +59 -0
  306. data/pkg/debian/debian/preinst.ex +37 -0
  307. data/pkg/debian/debian/prerm.ex +40 -0
  308. data/pkg/debian/debian/release.conf +5 -0
  309. data/pkg/debian/debian/rules +80 -0
  310. data/pkg/debian/debian/watch.ex +22 -0
  311. data/pkg/logrotate.conf +8 -0
  312. data/pkg/logstash-web.default +41 -0
  313. data/pkg/logstash-web.sysv.debian +201 -0
  314. data/pkg/logstash-web.upstart.ubuntu +18 -0
  315. data/pkg/logstash.default +45 -0
  316. data/pkg/logstash.sysv.debian +202 -0
  317. data/pkg/logstash.sysv.redhat +158 -0
  318. data/pkg/logstash.upstart.ubuntu +20 -0
  319. data/pkg/rpm/SOURCES/logstash.conf +26 -0
  320. data/pkg/rpm/SOURCES/logstash.init +80 -0
  321. data/pkg/rpm/SOURCES/logstash.logrotate +8 -0
  322. data/pkg/rpm/SOURCES/logstash.sysconfig +3 -0
  323. data/pkg/rpm/SOURCES/logstash.wrapper +105 -0
  324. data/pkg/rpm/SPECS/logstash.spec +180 -0
  325. data/pkg/rpm/readme.md +4 -0
  326. data/pkg/ubuntu/after-install.sh +7 -0
  327. data/pkg/ubuntu/before-install.sh +12 -0
  328. data/pkg/ubuntu/before-remove.sh +13 -0
  329. data/pull_release_note.rb +25 -0
  330. data/require-analyze.rb +22 -0
  331. data/spec/README.md +14 -0
  332. data/spec/codecs/edn.rb +40 -0
  333. data/spec/codecs/edn_lines.rb +53 -0
  334. data/spec/codecs/graphite.rb +96 -0
  335. data/spec/codecs/json.rb +57 -0
  336. data/spec/codecs/json_lines.rb +51 -0
  337. data/spec/codecs/json_spooler.rb +43 -0
  338. data/spec/codecs/msgpack.rb +39 -0
  339. data/spec/codecs/multiline.rb +60 -0
  340. data/spec/codecs/oldlogstashjson.rb +55 -0
  341. data/spec/codecs/plain.rb +35 -0
  342. data/spec/codecs/spool.rb +35 -0
  343. data/spec/conditionals/test.rb +323 -0
  344. data/spec/config.rb +31 -0
  345. data/spec/event.rb +165 -0
  346. data/spec/examples/fail2ban.rb +28 -0
  347. data/spec/examples/graphite-input.rb +41 -0
  348. data/spec/examples/mysql-slow-query.rb +70 -0
  349. data/spec/examples/parse-apache-logs.rb +66 -0
  350. data/spec/examples/parse-haproxy-logs.rb +115 -0
  351. data/spec/examples/syslog.rb +48 -0
  352. data/spec/filters/alter.rb +96 -0
  353. data/spec/filters/anonymize.rb +189 -0
  354. data/spec/filters/checksum.rb +41 -0
  355. data/spec/filters/clone.rb +67 -0
  356. data/spec/filters/collate.rb +122 -0
  357. data/spec/filters/csv.rb +174 -0
  358. data/spec/filters/date.rb +285 -0
  359. data/spec/filters/date_performance.rb +31 -0
  360. data/spec/filters/dns.rb +159 -0
  361. data/spec/filters/drop.rb +19 -0
  362. data/spec/filters/elapsed.rb +294 -0
  363. data/spec/filters/environment.rb +43 -0
  364. data/spec/filters/geoip.rb +62 -0
  365. data/spec/filters/grep.rb +342 -0
  366. data/spec/filters/grok.rb +473 -0
  367. data/spec/filters/grok/timeout2.rb +56 -0
  368. data/spec/filters/grok/timeouts.rb +39 -0
  369. data/spec/filters/i18n.rb +25 -0
  370. data/spec/filters/json.rb +72 -0
  371. data/spec/filters/json_encode.rb +37 -0
  372. data/spec/filters/kv.rb +403 -0
  373. data/spec/filters/metrics.rb +212 -0
  374. data/spec/filters/multiline.rb +119 -0
  375. data/spec/filters/mutate.rb +180 -0
  376. data/spec/filters/noop.rb +221 -0
  377. data/spec/filters/prune.rb +441 -0
  378. data/spec/filters/punct.rb +18 -0
  379. data/spec/filters/railsparallelrequest.rb +112 -0
  380. data/spec/filters/range.rb +169 -0
  381. data/spec/filters/split.rb +58 -0
  382. data/spec/filters/translate.rb +70 -0
  383. data/spec/filters/unique.rb +25 -0
  384. data/spec/filters/useragent.rb +42 -0
  385. data/spec/filters/xml.rb +157 -0
  386. data/spec/inputs/file.rb +107 -0
  387. data/spec/inputs/gelf.rb +52 -0
  388. data/spec/inputs/generator.rb +30 -0
  389. data/spec/inputs/imap.rb +60 -0
  390. data/spec/inputs/redis.rb +63 -0
  391. data/spec/inputs/relp.rb +70 -0
  392. data/spec/inputs/tcp.rb +101 -0
  393. data/spec/jar.rb +21 -0
  394. data/spec/outputs/csv.rb +266 -0
  395. data/spec/outputs/elasticsearch.rb +161 -0
  396. data/spec/outputs/elasticsearch_http.rb +240 -0
  397. data/spec/outputs/email.rb +173 -0
  398. data/spec/outputs/file.rb +82 -0
  399. data/spec/outputs/graphite.rb +236 -0
  400. data/spec/outputs/redis.rb +127 -0
  401. data/spec/speed.rb +20 -0
  402. data/spec/sqlite-test.rb +81 -0
  403. data/spec/support/LOGSTASH-733.rb +21 -0
  404. data/spec/support/LOGSTASH-820.rb +25 -0
  405. data/spec/support/akamai-grok.rb +26 -0
  406. data/spec/support/date-http.rb +17 -0
  407. data/spec/support/postwait1.rb +26 -0
  408. data/spec/support/pull375.rb +21 -0
  409. data/spec/test_utils.rb +125 -0
  410. data/spec/util/fieldeval_spec.rb +44 -0
  411. data/test/jenkins/config.xml.erb +74 -0
  412. data/test/jenkins/create-jobs.rb +23 -0
  413. data/test/jenkins/generatorjob.config.xml +66 -0
  414. data/tools/Gemfile +14 -0
  415. data/tools/Gemfile.jruby-1.9.lock +322 -0
  416. data/tools/Gemfile.rbx-2.1.lock +516 -0
  417. data/tools/Gemfile.ruby-1.9.1.lock +310 -0
  418. data/tools/Gemfile.ruby-2.0.0.lock +310 -0
  419. metadata +629 -0
@@ -0,0 +1,17 @@
1
+ logstash (1.1.10.dev~1) unstable; urgency=low
2
+
3
+ * Dev builds of master.
4
+
5
+ -- Jordan Sissel <jls@semicomplete.com> Tue, 2 Apr 2013 21:55:33 -0700
6
+
7
+ logstash (1.1.9~1) unstable; urgency=low
8
+
9
+ * Depend on openjdk. Don't start daemon by default.
10
+
11
+ -- Rudy Gevaert <Rudy.Gevaert@UGent.be> Wed, 27 Feb 2013 21:54:41 +0100
12
+
13
+ logstash (1.1.9) unstable; urgency=low
14
+
15
+ * Initial packaging release
16
+
17
+ -- Corey Quinn <corey@sequestered.net> Sun, 24 Feb 2013 18:24:41 -0800
@@ -0,0 +1 @@
1
+ 5
@@ -0,0 +1,16 @@
1
+ Source: logstash
2
+ Section: devel
3
+ Priority: extra
4
+ Maintainer: Corey Quinn <corey@sequestered.net>
5
+ Build-Depends: debhelper (>= 5)
6
+ Standards-Version: 3.7.2
7
+ Homepage: http://logstash.net
8
+
9
+ Package: logstash
10
+ Architecture: all
11
+ Depends: ${shlibs:Depends}, ${misc:Depends}, daemon, adduser, psmisc, default-jre
12
+ Description: tool for managing events and logs
13
+ logstash is a tool for managing events and logs. You can use it to collect logs,
14
+ parse them, and store them for later use (like, for searching). Speaking of
15
+ searching, logstash comes with a web interface for searching and drilling into
16
+ all of your logs.
@@ -0,0 +1,27 @@
1
+ This package was debianized by Corey Quinn <corey@sequestered.net>.
2
+
3
+ License:
4
+
5
+ This package is free software; you can redistribute it and/or modify
6
+ it under the terms of the GNU General Public License as published by
7
+ the Free Software Foundation; either version 2 of the License, or
8
+ (at your option) any later version.
9
+
10
+ This package is distributed in the hope that it will be useful,
11
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ GNU General Public License for more details.
14
+
15
+ You should have received a copy of the GNU General Public License
16
+ along with this package; if not, write to the Free Software
17
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18
+
19
+ On Debian systems, the complete text of the GNU General
20
+ Public License can be found in `/usr/share/common-licenses/GPL'.
21
+
22
+ The Debian packaging is (C) 2013, Corey Quinn <corey@sequestered.net> and
23
+ is licensed under the GPL, see above.
24
+
25
+
26
+ # Please also look if there are files or directories which have a
27
+ # different copyright/license attached and list them here.
@@ -0,0 +1,19 @@
1
+ # moved from rules install
2
+ usr/share/logstash
3
+
4
+ # this is where LS_HOME is stored, so let's leave it as is for now
5
+ var/lib/logstash
6
+
7
+ # Need to create logstash's own directory to track the pid since the daemon is not run as root
8
+ # and doesn't have permission to write the pid to /var/run (which would be the preferred location).
9
+ var/run/logstash
10
+
11
+ # Store logstash log file in it's own directory since they can become rather large and in the future
12
+ # rotating logs can be easily added.
13
+ var/log/logstash
14
+
15
+ # Cache directory for the unpacked logstash.jar file.
16
+ var/cache/logstash
17
+
18
+ # Temp dir for java
19
+ var/logstash/
File without changes
@@ -0,0 +1,39 @@
1
+ # defaults for logstash
2
+
3
+ # Start logstash on boot?
4
+ START=no
5
+
6
+ # pulled in from the init script; makes things easier.
7
+ NAME=logstash
8
+
9
+ # location of java
10
+ JAVA=/usr/bin/java
11
+
12
+ # arguments to pass to java
13
+ LS_JAVA_OPTS="-Xmx256m -Djava.io.tmpdir=/var/lib/logstash/"
14
+
15
+ PIDFILE=/var/run/logstash.pid
16
+
17
+ # user id to be invoked as
18
+ LS_USER=logstash
19
+
20
+ # location of the logstas jar file
21
+ LS_JAR=/usr/share/logstash/logstash.jar
22
+
23
+ # logstash home location
24
+ LS_HOME=/var/lib/logstash
25
+
26
+ # logstash log directory
27
+ LOG_DIR=/var/log/logstash
28
+
29
+ # logstash log file
30
+ LOG_FILE=$LOG_DIR/$NAME.log
31
+
32
+ # logstash configuration directory
33
+ CONF_DIR=/etc/logstash/conf.d
34
+
35
+ # Open file limit
36
+ OPEN_FILES=2048
37
+
38
+ # Nice level
39
+ NICE=19
@@ -0,0 +1,201 @@
1
+ #!/bin/bash
2
+ #
3
+ # /etc/init.d/logstash -- startup script for LogStash.
4
+ #
5
+ ### BEGIN INIT INFO
6
+ # Provides: logstash
7
+ # Required-Start: $all
8
+ # Required-Stop: $all
9
+ # Default-Start: 2 3 4 5
10
+ # Default-Stop: 0 1 6
11
+ # Short-Description: Starts logstash
12
+ # Description: Starts logstash using start-stop-daemon
13
+ ### END INIT INFO
14
+
15
+ set -e
16
+
17
+ PATH=/bin:/usr/bin:/sbin:/usr/sbin
18
+ NAME=logstash
19
+ DESC="Logstash Daemon"
20
+ DEFAULT=/etc/default/$NAME
21
+
22
+ if [ `id -u` -ne 0 ]; then
23
+ echo "You need root privileges to run this script"
24
+ exit 1
25
+ fi
26
+
27
+ . /lib/lsb/init-functions
28
+
29
+ if [ -r /etc/default/rcS ]; then
30
+ . /etc/default/rcS
31
+ fi
32
+
33
+ # The following variables can be overwritten in $DEFAULT
34
+
35
+ # Run logstash as this user ID and group ID
36
+ LS_USER=logstash
37
+ LS_GROUP=logstash
38
+
39
+ JAVA=/usr/bin/java
40
+
41
+ # Directory where the logstash all in one jar lives
42
+ LS_HOME=/var/lib/logstash
43
+
44
+ # Additional Java OPTS
45
+ LS_JAVA_OPTS=" -Djava.io.tmpdir=/var/logstash/"
46
+
47
+ # logstash log directory
48
+ LOG_DIR=/var/log/logstash
49
+
50
+ # logstash configuration directory
51
+ CONF_DIR=/etc/logstash/conf.d
52
+
53
+ # logstash log file
54
+ LOG_FILE=$LOG_DIR/$NAME.log
55
+
56
+ # Open File limit
57
+ OPEN_FILES=2048
58
+
59
+
60
+
61
+ # Nice level
62
+ NICE=19
63
+
64
+ # End of variables that can be overwritten in $DEFAULT
65
+
66
+ # overwrite settings from default file
67
+ if [ -f "$DEFAULT" ]; then
68
+ . "$DEFAULT"
69
+ fi
70
+
71
+ # Define other required variables
72
+ PID_FILE=/var/run/$NAME.pid
73
+ DAEMON=$LS_JAR
74
+ DAEMON_OPTS="agent -f ${CONF_DIR} --log ${LOG_FILE}"
75
+
76
+ is_true() {
77
+ if [ "x$1" = "xtrue" -o "x$1" = "xyes" -o "x$1" = "x1" ] ; then
78
+ return 0
79
+ else
80
+ return 1
81
+ fi
82
+ }
83
+
84
+ # Check DAEMON exists
85
+ if ! test -e $DAEMON; then
86
+ log_failure_msg "Daemon $DAEMON doesn't exist"
87
+ exit 1
88
+ fi
89
+
90
+ case "$1" in
91
+ start)
92
+ if ! is_true "$START" ; then
93
+ echo "logstash not configured to start, please edit /etc/default/logstash to enable"
94
+ exit 0
95
+ fi
96
+
97
+ if [ -z "$JAVA" ]; then
98
+ log_failure_msg "no JDK found - $JAVA"
99
+ exit 1
100
+ fi
101
+
102
+ # Check if a config file exists
103
+ if ! test -e $CONF_DIR/*.conf; then
104
+ log_failure_msg "There aren't any configuration files in $CONF_DIR"
105
+ exit 1
106
+ fi
107
+
108
+ log_daemon_msg "Starting $DESC"
109
+
110
+ if start-stop-daemon --test --start --pidfile "$PID_FILE" \
111
+ --user "$LS_USER" --exec "$JAVA" \
112
+ >/dev/null; then
113
+ # Prepare environment
114
+ ulimit -n $OPEN_FILES
115
+
116
+ # Start Daemon
117
+ start-stop-daemon --start -b --user "$LS_USER" -c "$LS_USER":"$LS_GROUP" \
118
+ -d "$LS_HOME" --pidfile "$PID_FILE" --make-pidfile \
119
+ --exec "$JAVA" -- $LS_JAVA_OPTS -jar $DAEMON $DAEMON_OPTS
120
+
121
+ sleep 1
122
+
123
+ if start-stop-daemon --test --start --pidfile "$PID_FILE" \
124
+ --user "$LS_USER" --exec "$JAVA" \
125
+ >/dev/null; then
126
+
127
+ if [ -f "$PID_FILE" ]; then
128
+ rm -f "$PID_FILE"
129
+ fi
130
+
131
+ log_end_msg 1
132
+ else
133
+ log_end_msg 0
134
+ fi
135
+ else
136
+ log_progress_msg "(already running)"
137
+ log_end_msg 0
138
+ fi
139
+ ;;
140
+ stop)
141
+ log_daemon_msg "Stopping $DESC"
142
+
143
+ set +e
144
+
145
+ if [ -f "$PID_FILE" ]; then
146
+ start-stop-daemon --stop --pidfile "$PID_FILE" \
147
+ --user "$LS_USER" \
148
+ --retry=TERM/20/KILL/5 >/dev/null
149
+
150
+ if [ $? -eq 1 ]; then
151
+ log_progress_msg "$DESC is not running but pid file exists, cleaning up"
152
+ elif [ $? -eq 3 ]; then
153
+ PID="`cat $PID_FILE`"
154
+ log_failure_msg "Failed to stop $DESC (pid $PID)"
155
+ exit 1
156
+ fi
157
+
158
+ rm -f "$PID_FILE"
159
+ else
160
+ log_progress_msg "(not running)"
161
+ fi
162
+
163
+ log_end_msg 0
164
+ set -e
165
+ ;;
166
+ status)
167
+ set +e
168
+
169
+ start-stop-daemon --test --start --pidfile "$PID_FILE" \
170
+ --user "$LS_USER" --exec "$JAVA" \
171
+ >/dev/null 2>&1
172
+
173
+ if [ "$?" = "0" ]; then
174
+ if [ -f "$PID_FILE" ]; then
175
+ log_success_msg "$DESC is not running, but pid file exists."
176
+ exit 1
177
+ else
178
+ log_success_msg "$DESC is not running."
179
+ exit 3
180
+ fi
181
+ else
182
+ log_success_msg "$DESC is running with pid `cat $PID_FILE`"
183
+ fi
184
+
185
+ set -e
186
+ ;;
187
+ restart|force-reload)
188
+ if [ -f "$PID_FILE" ]; then
189
+ $0 stop
190
+ sleep 1
191
+ fi
192
+
193
+ $0 start
194
+ ;;
195
+ *)
196
+ log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
197
+ exit 1
198
+ ;;
199
+ esac
200
+
201
+ exit 0
@@ -0,0 +1 @@
1
+ logstash.jar usr/share/logstash
@@ -0,0 +1,9 @@
1
+ /var/log/logstash/logstash.log {
2
+ weekly
3
+ copytruncate
4
+ missingok
5
+ rotate 52
6
+ compress
7
+ delaycompress
8
+ notifempty
9
+ }
@@ -0,0 +1,68 @@
1
+ #!/bin/sh
2
+ # postinst script for jenkins
3
+ #
4
+ # see: dh_installdeb(1)
5
+
6
+ set -e
7
+
8
+ # summary of how this script can be called:
9
+ # * <postinst> `configure' <most-recently-configured-version>
10
+ # * <old-postinst> `abort-upgrade' <new version>
11
+ # * <conflictor's-postinst> `abort-remove' `in-favour' <package>
12
+ # <new-version>
13
+ # * <postinst> `abort-remove'
14
+ # * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
15
+ # <failed-install-package> <version> `removing'
16
+ # <conflicting-package> <version>
17
+ # for details, see http://www.debian.org/doc/debian-policy/ or
18
+ # the debian-policy package
19
+
20
+ LS_GROUP=logstash
21
+ LS_USER=logstash
22
+ case "$1" in
23
+ configure)
24
+
25
+ [ -r /etc/default/logstash ] && . /etc/default/logstash
26
+ : ${LS_USER:=logstash}
27
+
28
+ # Create logstash user if it doesn't exist.
29
+ # sometimes tools that users want logstash to run need a shell,
30
+ # so use /bin/bash.
31
+ if ! getent group "$LS_GROUP" > /dev/null 2>&1 ; then
32
+ addgroup --system "$LS_GROUP" --quiet
33
+ fi
34
+
35
+ if ! id $LS_USER > /dev/null 2>&1 ; then
36
+ adduser --system --home /var/lib/logstash --no-create-home \
37
+ --ingroup $LS_GROUP --disabled-password --shell /bin/bash $LS_USER
38
+ fi
39
+
40
+ # directories needed for logstash
41
+ # we don't do -R because it can take a long time on big installation
42
+ chown $LS_USER:adm /var/lib/logstash /var/log/logstash
43
+ # we don't do "chmod 750" so that the user can choose the pemission for g and o on their own
44
+ chmod u+rwx /var/lib/logstash /var/log/logstash
45
+
46
+ # make sure logstash can delete everything in /var/cache/logstash to
47
+ # re-explode jar.
48
+
49
+ chown -R $LS_USER:adm /var/cache/logstash /var/run/logstash
50
+ chmod -R 750 /var/cache/logstash /var/run/logstash
51
+ ;;
52
+
53
+ abort-upgrade|abort-remove|abort-deconfigure)
54
+ ;;
55
+
56
+ *)
57
+ echo "postinst called with unknown argument \`$1'" >&2
58
+ exit 1
59
+ ;;
60
+ esac
61
+
62
+ # dh_installdeb will replace this with shell code automatically
63
+ # generated by other debhelper scripts.
64
+
65
+ #DEBHELPER#
66
+
67
+ exit 0
68
+
@@ -0,0 +1,23 @@
1
+ #!/bin/sh
2
+
3
+ set -e
4
+
5
+ case "$1" in
6
+ purge)
7
+ userdel logstash || true
8
+ rm -rf /var/lib/logstash /var/log/logstash \
9
+ /var/run/logstash /var/cache/logstash
10
+ ;;
11
+
12
+ remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
13
+ ;;
14
+
15
+ *)
16
+ echo "postrm called with unknown argument \`$1'" >&2
17
+ exit 1
18
+ ;;
19
+ esac
20
+
21
+ #DEBHELPER#
22
+
23
+ exit 0
@@ -0,0 +1,59 @@
1
+ .\" Hey, EMACS: -*- nroff -*-
2
+ .\" First parameter, NAME, should be all caps
3
+ .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
4
+ .\" other parameters are allowed: see man(7), man(1)
5
+ .TH LOGSTASH SECTION "April 25, 2008"
6
+ .\" Please adjust this date whenever revising the manpage.
7
+ .\"
8
+ .\" Some roff macros, for reference:
9
+ .\" .nh disable hyphenation
10
+ .\" .hy enable hyphenation
11
+ .\" .ad l left justify
12
+ .\" .ad b justify to both left and right margins
13
+ .\" .nf disable filling
14
+ .\" .fi enable filling
15
+ .\" .br insert line break
16
+ .\" .sp <n> insert n+1 empty lines
17
+ .\" for manpage-specific macros, see man(7)
18
+ .SH NAME
19
+ logstash \- program to do something
20
+ .SH SYNOPSIS
21
+ .B logstash
22
+ .RI [ options ] " files" ...
23
+ .br
24
+ .B bar
25
+ .RI [ options ] " files" ...
26
+ .SH DESCRIPTION
27
+ This manual page documents briefly the
28
+ .B logstash
29
+ and
30
+ .B bar
31
+ commands.
32
+ .PP
33
+ .\" TeX users may be more comfortable with the \fB<whatever>\fP and
34
+ .\" \fI<whatever>\fP escape sequences to invode bold face and italics,
35
+ .\" respectively.
36
+ \fBlogstash\fP is a program that...
37
+ .SH OPTIONS
38
+ These programs follow the usual GNU command line syntax, with long
39
+ options starting with two dashes (`-').
40
+ A summary of options is included below.
41
+ For a complete description, see the Info files.
42
+ .TP
43
+ .B \-h, \-\-help
44
+ Show summary of options.
45
+ .TP
46
+ .B \-v, \-\-version
47
+ Show version of program.
48
+ .SH SEE ALSO
49
+ .BR bar (1),
50
+ .BR baz (1).
51
+ .br
52
+ The programs are documented fully by
53
+ .IR "The Rise and Fall of a Fooish Bar" ,
54
+ available via the Info system.
55
+ .SH AUTHOR
56
+ logstash was written by <upstream author>.
57
+ .PP
58
+ This manual page was written by Kohsuke Kawaguchi <kk@kohsuke.org>,
59
+ for the Debian project (but may be used by others).