logstash-lib 1.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (419) hide show
  1. data/.gitignore +24 -0
  2. data/.tailor +8 -0
  3. data/.travis.yml +12 -0
  4. data/CHANGELOG +1185 -0
  5. data/CONTRIBUTING.md +61 -0
  6. data/CONTRIBUTORS +79 -0
  7. data/LICENSE +14 -0
  8. data/Makefile +460 -0
  9. data/README.md +120 -0
  10. data/STYLE.md +96 -0
  11. data/bin/logstash +37 -0
  12. data/bin/logstash-test +4 -0
  13. data/bin/logstash-web +4 -0
  14. data/bin/logstash.lib.sh +78 -0
  15. data/bot/check_pull_changelog.rb +89 -0
  16. data/docs/configuration.md +260 -0
  17. data/docs/docgen.rb +242 -0
  18. data/docs/extending/example-add-a-new-filter.md +121 -0
  19. data/docs/extending/index.md +91 -0
  20. data/docs/flags.md +43 -0
  21. data/docs/generate_index.rb +28 -0
  22. data/docs/index.html.erb +56 -0
  23. data/docs/learn.md +46 -0
  24. data/docs/life-of-an-event.md +109 -0
  25. data/docs/logging-tool-comparisons.md +60 -0
  26. data/docs/plugin-doc.html.erb +91 -0
  27. data/docs/plugin-milestones.md +41 -0
  28. data/docs/plugin-synopsis.html.erb +24 -0
  29. data/docs/release-engineering.md +46 -0
  30. data/docs/release-test-results.md +14 -0
  31. data/docs/repositories.md +35 -0
  32. data/docs/tutorials/10-minute-walkthrough/apache-elasticsearch.conf +35 -0
  33. data/docs/tutorials/10-minute-walkthrough/apache-parse.conf +33 -0
  34. data/docs/tutorials/10-minute-walkthrough/apache_log.1 +1 -0
  35. data/docs/tutorials/10-minute-walkthrough/apache_log.2.bz2 +0 -0
  36. data/docs/tutorials/10-minute-walkthrough/hello-search.conf +25 -0
  37. data/docs/tutorials/10-minute-walkthrough/hello.conf +16 -0
  38. data/docs/tutorials/10-minute-walkthrough/index.md +124 -0
  39. data/docs/tutorials/10-minute-walkthrough/step-5-output.txt +17 -0
  40. data/docs/tutorials/getting-started-centralized-overview-diagram.png +0 -0
  41. data/docs/tutorials/getting-started-centralized-overview-diagram.xml +1 -0
  42. data/docs/tutorials/getting-started-centralized.md +217 -0
  43. data/docs/tutorials/getting-started-simple.md +200 -0
  44. data/docs/tutorials/just-enough-rabbitmq-for-logstash.md +201 -0
  45. data/docs/tutorials/media/frontend-response-codes.png +0 -0
  46. data/docs/tutorials/metrics-from-logs.md +84 -0
  47. data/docs/tutorials/zeromq.md +118 -0
  48. data/extract_services.rb +29 -0
  49. data/gembag.rb +64 -0
  50. data/lib/logstash-event.rb +2 -0
  51. data/lib/logstash.rb +4 -0
  52. data/lib/logstash/JRUBY-6970-openssl.rb +22 -0
  53. data/lib/logstash/JRUBY-6970.rb +102 -0
  54. data/lib/logstash/agent.rb +305 -0
  55. data/lib/logstash/certs/cacert.pem +3895 -0
  56. data/lib/logstash/codecs/base.rb +49 -0
  57. data/lib/logstash/codecs/compress_spooler.rb +50 -0
  58. data/lib/logstash/codecs/dots.rb +18 -0
  59. data/lib/logstash/codecs/edn.rb +28 -0
  60. data/lib/logstash/codecs/edn_lines.rb +36 -0
  61. data/lib/logstash/codecs/fluent.rb +55 -0
  62. data/lib/logstash/codecs/graphite.rb +114 -0
  63. data/lib/logstash/codecs/json.rb +41 -0
  64. data/lib/logstash/codecs/json_lines.rb +52 -0
  65. data/lib/logstash/codecs/json_spooler.rb +22 -0
  66. data/lib/logstash/codecs/line.rb +58 -0
  67. data/lib/logstash/codecs/msgpack.rb +43 -0
  68. data/lib/logstash/codecs/multiline.rb +189 -0
  69. data/lib/logstash/codecs/netflow.rb +342 -0
  70. data/lib/logstash/codecs/netflow/util.rb +212 -0
  71. data/lib/logstash/codecs/noop.rb +19 -0
  72. data/lib/logstash/codecs/oldlogstashjson.rb +56 -0
  73. data/lib/logstash/codecs/plain.rb +48 -0
  74. data/lib/logstash/codecs/rubydebug.rb +22 -0
  75. data/lib/logstash/codecs/spool.rb +38 -0
  76. data/lib/logstash/config/Makefile +4 -0
  77. data/lib/logstash/config/config_ast.rb +380 -0
  78. data/lib/logstash/config/file.rb +39 -0
  79. data/lib/logstash/config/grammar.rb +3504 -0
  80. data/lib/logstash/config/grammar.treetop +241 -0
  81. data/lib/logstash/config/mixin.rb +464 -0
  82. data/lib/logstash/config/registry.rb +13 -0
  83. data/lib/logstash/config/test.conf +18 -0
  84. data/lib/logstash/errors.rb +10 -0
  85. data/lib/logstash/event.rb +262 -0
  86. data/lib/logstash/filters/advisor.rb +178 -0
  87. data/lib/logstash/filters/alter.rb +173 -0
  88. data/lib/logstash/filters/anonymize.rb +93 -0
  89. data/lib/logstash/filters/base.rb +190 -0
  90. data/lib/logstash/filters/checksum.rb +50 -0
  91. data/lib/logstash/filters/cidr.rb +76 -0
  92. data/lib/logstash/filters/cipher.rb +145 -0
  93. data/lib/logstash/filters/clone.rb +35 -0
  94. data/lib/logstash/filters/collate.rb +114 -0
  95. data/lib/logstash/filters/csv.rb +94 -0
  96. data/lib/logstash/filters/date.rb +244 -0
  97. data/lib/logstash/filters/dns.rb +201 -0
  98. data/lib/logstash/filters/drop.rb +32 -0
  99. data/lib/logstash/filters/elapsed.rb +256 -0
  100. data/lib/logstash/filters/elasticsearch.rb +73 -0
  101. data/lib/logstash/filters/environment.rb +27 -0
  102. data/lib/logstash/filters/extractnumbers.rb +84 -0
  103. data/lib/logstash/filters/gelfify.rb +52 -0
  104. data/lib/logstash/filters/geoip.rb +145 -0
  105. data/lib/logstash/filters/grep.rb +153 -0
  106. data/lib/logstash/filters/grok.rb +425 -0
  107. data/lib/logstash/filters/grokdiscovery.rb +75 -0
  108. data/lib/logstash/filters/i18n.rb +51 -0
  109. data/lib/logstash/filters/json.rb +90 -0
  110. data/lib/logstash/filters/json_encode.rb +52 -0
  111. data/lib/logstash/filters/kv.rb +232 -0
  112. data/lib/logstash/filters/metaevent.rb +68 -0
  113. data/lib/logstash/filters/metrics.rb +237 -0
  114. data/lib/logstash/filters/multiline.rb +241 -0
  115. data/lib/logstash/filters/mutate.rb +399 -0
  116. data/lib/logstash/filters/noop.rb +21 -0
  117. data/lib/logstash/filters/prune.rb +149 -0
  118. data/lib/logstash/filters/punct.rb +32 -0
  119. data/lib/logstash/filters/railsparallelrequest.rb +86 -0
  120. data/lib/logstash/filters/range.rb +142 -0
  121. data/lib/logstash/filters/ruby.rb +42 -0
  122. data/lib/logstash/filters/sleep.rb +111 -0
  123. data/lib/logstash/filters/split.rb +64 -0
  124. data/lib/logstash/filters/sumnumbers.rb +73 -0
  125. data/lib/logstash/filters/syslog_pri.rb +107 -0
  126. data/lib/logstash/filters/translate.rb +121 -0
  127. data/lib/logstash/filters/unique.rb +29 -0
  128. data/lib/logstash/filters/urldecode.rb +57 -0
  129. data/lib/logstash/filters/useragent.rb +112 -0
  130. data/lib/logstash/filters/uuid.rb +58 -0
  131. data/lib/logstash/filters/xml.rb +139 -0
  132. data/lib/logstash/filters/zeromq.rb +123 -0
  133. data/lib/logstash/filterworker.rb +122 -0
  134. data/lib/logstash/inputs/base.rb +125 -0
  135. data/lib/logstash/inputs/collectd.rb +306 -0
  136. data/lib/logstash/inputs/drupal_dblog.rb +323 -0
  137. data/lib/logstash/inputs/drupal_dblog/jdbcconnection.rb +66 -0
  138. data/lib/logstash/inputs/elasticsearch.rb +140 -0
  139. data/lib/logstash/inputs/eventlog.rb +129 -0
  140. data/lib/logstash/inputs/eventlog/racob_fix.rb +44 -0
  141. data/lib/logstash/inputs/exec.rb +69 -0
  142. data/lib/logstash/inputs/file.rb +146 -0
  143. data/lib/logstash/inputs/ganglia.rb +127 -0
  144. data/lib/logstash/inputs/ganglia/gmondpacket.rb +146 -0
  145. data/lib/logstash/inputs/ganglia/xdr.rb +327 -0
  146. data/lib/logstash/inputs/gelf.rb +138 -0
  147. data/lib/logstash/inputs/gemfire.rb +222 -0
  148. data/lib/logstash/inputs/generator.rb +97 -0
  149. data/lib/logstash/inputs/graphite.rb +41 -0
  150. data/lib/logstash/inputs/heroku.rb +51 -0
  151. data/lib/logstash/inputs/imap.rb +136 -0
  152. data/lib/logstash/inputs/irc.rb +84 -0
  153. data/lib/logstash/inputs/log4j.rb +136 -0
  154. data/lib/logstash/inputs/lumberjack.rb +53 -0
  155. data/lib/logstash/inputs/pipe.rb +57 -0
  156. data/lib/logstash/inputs/rabbitmq.rb +126 -0
  157. data/lib/logstash/inputs/rabbitmq/bunny.rb +118 -0
  158. data/lib/logstash/inputs/rabbitmq/hot_bunnies.rb +1 -0
  159. data/lib/logstash/inputs/rabbitmq/march_hare.rb +129 -0
  160. data/lib/logstash/inputs/redis.rb +263 -0
  161. data/lib/logstash/inputs/relp.rb +106 -0
  162. data/lib/logstash/inputs/s3.rb +279 -0
  163. data/lib/logstash/inputs/snmptrap.rb +87 -0
  164. data/lib/logstash/inputs/sqlite.rb +185 -0
  165. data/lib/logstash/inputs/sqs.rb +172 -0
  166. data/lib/logstash/inputs/stdin.rb +46 -0
  167. data/lib/logstash/inputs/stomp.rb +84 -0
  168. data/lib/logstash/inputs/syslog.rb +237 -0
  169. data/lib/logstash/inputs/tcp.rb +231 -0
  170. data/lib/logstash/inputs/threadable.rb +18 -0
  171. data/lib/logstash/inputs/twitter.rb +82 -0
  172. data/lib/logstash/inputs/udp.rb +81 -0
  173. data/lib/logstash/inputs/unix.rb +163 -0
  174. data/lib/logstash/inputs/varnishlog.rb +48 -0
  175. data/lib/logstash/inputs/websocket.rb +50 -0
  176. data/lib/logstash/inputs/wmi.rb +72 -0
  177. data/lib/logstash/inputs/xmpp.rb +81 -0
  178. data/lib/logstash/inputs/zenoss.rb +143 -0
  179. data/lib/logstash/inputs/zeromq.rb +165 -0
  180. data/lib/logstash/kibana.rb +113 -0
  181. data/lib/logstash/loadlibs.rb +9 -0
  182. data/lib/logstash/logging.rb +89 -0
  183. data/lib/logstash/monkeypatches-for-bugs.rb +2 -0
  184. data/lib/logstash/monkeypatches-for-debugging.rb +47 -0
  185. data/lib/logstash/monkeypatches-for-performance.rb +66 -0
  186. data/lib/logstash/multiqueue.rb +53 -0
  187. data/lib/logstash/namespace.rb +16 -0
  188. data/lib/logstash/outputs/base.rb +120 -0
  189. data/lib/logstash/outputs/boundary.rb +116 -0
  190. data/lib/logstash/outputs/circonus.rb +78 -0
  191. data/lib/logstash/outputs/cloudwatch.rb +351 -0
  192. data/lib/logstash/outputs/csv.rb +55 -0
  193. data/lib/logstash/outputs/datadog.rb +93 -0
  194. data/lib/logstash/outputs/datadog_metrics.rb +123 -0
  195. data/lib/logstash/outputs/elasticsearch.rb +332 -0
  196. data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +44 -0
  197. data/lib/logstash/outputs/elasticsearch_http.rb +256 -0
  198. data/lib/logstash/outputs/elasticsearch_river.rb +214 -0
  199. data/lib/logstash/outputs/email.rb +299 -0
  200. data/lib/logstash/outputs/exec.rb +40 -0
  201. data/lib/logstash/outputs/file.rb +180 -0
  202. data/lib/logstash/outputs/ganglia.rb +75 -0
  203. data/lib/logstash/outputs/gelf.rb +208 -0
  204. data/lib/logstash/outputs/gemfire.rb +103 -0
  205. data/lib/logstash/outputs/google_bigquery.rb +570 -0
  206. data/lib/logstash/outputs/google_cloud_storage.rb +431 -0
  207. data/lib/logstash/outputs/graphite.rb +143 -0
  208. data/lib/logstash/outputs/graphtastic.rb +185 -0
  209. data/lib/logstash/outputs/hipchat.rb +80 -0
  210. data/lib/logstash/outputs/http.rb +142 -0
  211. data/lib/logstash/outputs/irc.rb +80 -0
  212. data/lib/logstash/outputs/jira.rb +109 -0
  213. data/lib/logstash/outputs/juggernaut.rb +105 -0
  214. data/lib/logstash/outputs/librato.rb +146 -0
  215. data/lib/logstash/outputs/loggly.rb +93 -0
  216. data/lib/logstash/outputs/lumberjack.rb +51 -0
  217. data/lib/logstash/outputs/metriccatcher.rb +103 -0
  218. data/lib/logstash/outputs/mongodb.rb +81 -0
  219. data/lib/logstash/outputs/nagios.rb +119 -0
  220. data/lib/logstash/outputs/nagios_nsca.rb +123 -0
  221. data/lib/logstash/outputs/null.rb +18 -0
  222. data/lib/logstash/outputs/opentsdb.rb +101 -0
  223. data/lib/logstash/outputs/pagerduty.rb +79 -0
  224. data/lib/logstash/outputs/pipe.rb +132 -0
  225. data/lib/logstash/outputs/rabbitmq.rb +96 -0
  226. data/lib/logstash/outputs/rabbitmq/bunny.rb +135 -0
  227. data/lib/logstash/outputs/rabbitmq/hot_bunnies.rb +1 -0
  228. data/lib/logstash/outputs/rabbitmq/march_hare.rb +143 -0
  229. data/lib/logstash/outputs/redis.rb +245 -0
  230. data/lib/logstash/outputs/riak.rb +152 -0
  231. data/lib/logstash/outputs/riemann.rb +109 -0
  232. data/lib/logstash/outputs/s3.rb +356 -0
  233. data/lib/logstash/outputs/sns.rb +124 -0
  234. data/lib/logstash/outputs/solr_http.rb +78 -0
  235. data/lib/logstash/outputs/sqs.rb +141 -0
  236. data/lib/logstash/outputs/statsd.rb +116 -0
  237. data/lib/logstash/outputs/stdout.rb +53 -0
  238. data/lib/logstash/outputs/stomp.rb +67 -0
  239. data/lib/logstash/outputs/syslog.rb +145 -0
  240. data/lib/logstash/outputs/tcp.rb +145 -0
  241. data/lib/logstash/outputs/udp.rb +38 -0
  242. data/lib/logstash/outputs/websocket.rb +46 -0
  243. data/lib/logstash/outputs/websocket/app.rb +29 -0
  244. data/lib/logstash/outputs/websocket/pubsub.rb +45 -0
  245. data/lib/logstash/outputs/xmpp.rb +78 -0
  246. data/lib/logstash/outputs/zabbix.rb +108 -0
  247. data/lib/logstash/outputs/zeromq.rb +125 -0
  248. data/lib/logstash/pipeline.rb +286 -0
  249. data/lib/logstash/plugin.rb +150 -0
  250. data/lib/logstash/plugin_mixins/aws_config.rb +93 -0
  251. data/lib/logstash/program.rb +15 -0
  252. data/lib/logstash/runner.rb +238 -0
  253. data/lib/logstash/sized_queue.rb +8 -0
  254. data/lib/logstash/test.rb +183 -0
  255. data/lib/logstash/threadwatchdog.rb +37 -0
  256. data/lib/logstash/time_addon.rb +33 -0
  257. data/lib/logstash/util.rb +106 -0
  258. data/lib/logstash/util/buftok.rb +139 -0
  259. data/lib/logstash/util/charset.rb +39 -0
  260. data/lib/logstash/util/fieldreference.rb +50 -0
  261. data/lib/logstash/util/password.rb +25 -0
  262. data/lib/logstash/util/prctl.rb +11 -0
  263. data/lib/logstash/util/relp.rb +326 -0
  264. data/lib/logstash/util/require-helper.rb +18 -0
  265. data/lib/logstash/util/socket_peer.rb +7 -0
  266. data/lib/logstash/util/zenoss.rb +566 -0
  267. data/lib/logstash/util/zeromq.rb +47 -0
  268. data/lib/logstash/version.rb +6 -0
  269. data/locales/en.yml +170 -0
  270. data/logstash-event.gemspec +29 -0
  271. data/logstash.gemspec +128 -0
  272. data/patterns/firewalls +60 -0
  273. data/patterns/grok-patterns +91 -0
  274. data/patterns/haproxy +37 -0
  275. data/patterns/java +3 -0
  276. data/patterns/linux-syslog +14 -0
  277. data/patterns/mcollective +1 -0
  278. data/patterns/mcollective-patterns +4 -0
  279. data/patterns/nagios +108 -0
  280. data/patterns/postgresql +3 -0
  281. data/patterns/redis +3 -0
  282. data/patterns/ruby +2 -0
  283. data/pkg/build.sh +135 -0
  284. data/pkg/centos/after-install.sh +1 -0
  285. data/pkg/centos/before-install.sh +10 -0
  286. data/pkg/centos/before-remove.sh +11 -0
  287. data/pkg/centos/sysconfig +15 -0
  288. data/pkg/debian/after-install.sh +5 -0
  289. data/pkg/debian/before-install.sh +13 -0
  290. data/pkg/debian/before-remove.sh +13 -0
  291. data/pkg/debian/build.sh +34 -0
  292. data/pkg/debian/debian/README +6 -0
  293. data/pkg/debian/debian/changelog +17 -0
  294. data/pkg/debian/debian/compat +1 -0
  295. data/pkg/debian/debian/control +16 -0
  296. data/pkg/debian/debian/copyright +27 -0
  297. data/pkg/debian/debian/dirs +19 -0
  298. data/pkg/debian/debian/docs +0 -0
  299. data/pkg/debian/debian/logstash.default +39 -0
  300. data/pkg/debian/debian/logstash.init +201 -0
  301. data/pkg/debian/debian/logstash.install +1 -0
  302. data/pkg/debian/debian/logstash.logrotate +9 -0
  303. data/pkg/debian/debian/logstash.postinst +68 -0
  304. data/pkg/debian/debian/logstash.postrm +23 -0
  305. data/pkg/debian/debian/manpage.1.ex +59 -0
  306. data/pkg/debian/debian/preinst.ex +37 -0
  307. data/pkg/debian/debian/prerm.ex +40 -0
  308. data/pkg/debian/debian/release.conf +5 -0
  309. data/pkg/debian/debian/rules +80 -0
  310. data/pkg/debian/debian/watch.ex +22 -0
  311. data/pkg/logrotate.conf +8 -0
  312. data/pkg/logstash-web.default +41 -0
  313. data/pkg/logstash-web.sysv.debian +201 -0
  314. data/pkg/logstash-web.upstart.ubuntu +18 -0
  315. data/pkg/logstash.default +45 -0
  316. data/pkg/logstash.sysv.debian +202 -0
  317. data/pkg/logstash.sysv.redhat +158 -0
  318. data/pkg/logstash.upstart.ubuntu +20 -0
  319. data/pkg/rpm/SOURCES/logstash.conf +26 -0
  320. data/pkg/rpm/SOURCES/logstash.init +80 -0
  321. data/pkg/rpm/SOURCES/logstash.logrotate +8 -0
  322. data/pkg/rpm/SOURCES/logstash.sysconfig +3 -0
  323. data/pkg/rpm/SOURCES/logstash.wrapper +105 -0
  324. data/pkg/rpm/SPECS/logstash.spec +180 -0
  325. data/pkg/rpm/readme.md +4 -0
  326. data/pkg/ubuntu/after-install.sh +7 -0
  327. data/pkg/ubuntu/before-install.sh +12 -0
  328. data/pkg/ubuntu/before-remove.sh +13 -0
  329. data/pull_release_note.rb +25 -0
  330. data/require-analyze.rb +22 -0
  331. data/spec/README.md +14 -0
  332. data/spec/codecs/edn.rb +40 -0
  333. data/spec/codecs/edn_lines.rb +53 -0
  334. data/spec/codecs/graphite.rb +96 -0
  335. data/spec/codecs/json.rb +57 -0
  336. data/spec/codecs/json_lines.rb +51 -0
  337. data/spec/codecs/json_spooler.rb +43 -0
  338. data/spec/codecs/msgpack.rb +39 -0
  339. data/spec/codecs/multiline.rb +60 -0
  340. data/spec/codecs/oldlogstashjson.rb +55 -0
  341. data/spec/codecs/plain.rb +35 -0
  342. data/spec/codecs/spool.rb +35 -0
  343. data/spec/conditionals/test.rb +323 -0
  344. data/spec/config.rb +31 -0
  345. data/spec/event.rb +165 -0
  346. data/spec/examples/fail2ban.rb +28 -0
  347. data/spec/examples/graphite-input.rb +41 -0
  348. data/spec/examples/mysql-slow-query.rb +70 -0
  349. data/spec/examples/parse-apache-logs.rb +66 -0
  350. data/spec/examples/parse-haproxy-logs.rb +115 -0
  351. data/spec/examples/syslog.rb +48 -0
  352. data/spec/filters/alter.rb +96 -0
  353. data/spec/filters/anonymize.rb +189 -0
  354. data/spec/filters/checksum.rb +41 -0
  355. data/spec/filters/clone.rb +67 -0
  356. data/spec/filters/collate.rb +122 -0
  357. data/spec/filters/csv.rb +174 -0
  358. data/spec/filters/date.rb +285 -0
  359. data/spec/filters/date_performance.rb +31 -0
  360. data/spec/filters/dns.rb +159 -0
  361. data/spec/filters/drop.rb +19 -0
  362. data/spec/filters/elapsed.rb +294 -0
  363. data/spec/filters/environment.rb +43 -0
  364. data/spec/filters/geoip.rb +62 -0
  365. data/spec/filters/grep.rb +342 -0
  366. data/spec/filters/grok.rb +473 -0
  367. data/spec/filters/grok/timeout2.rb +56 -0
  368. data/spec/filters/grok/timeouts.rb +39 -0
  369. data/spec/filters/i18n.rb +25 -0
  370. data/spec/filters/json.rb +72 -0
  371. data/spec/filters/json_encode.rb +37 -0
  372. data/spec/filters/kv.rb +403 -0
  373. data/spec/filters/metrics.rb +212 -0
  374. data/spec/filters/multiline.rb +119 -0
  375. data/spec/filters/mutate.rb +180 -0
  376. data/spec/filters/noop.rb +221 -0
  377. data/spec/filters/prune.rb +441 -0
  378. data/spec/filters/punct.rb +18 -0
  379. data/spec/filters/railsparallelrequest.rb +112 -0
  380. data/spec/filters/range.rb +169 -0
  381. data/spec/filters/split.rb +58 -0
  382. data/spec/filters/translate.rb +70 -0
  383. data/spec/filters/unique.rb +25 -0
  384. data/spec/filters/useragent.rb +42 -0
  385. data/spec/filters/xml.rb +157 -0
  386. data/spec/inputs/file.rb +107 -0
  387. data/spec/inputs/gelf.rb +52 -0
  388. data/spec/inputs/generator.rb +30 -0
  389. data/spec/inputs/imap.rb +60 -0
  390. data/spec/inputs/redis.rb +63 -0
  391. data/spec/inputs/relp.rb +70 -0
  392. data/spec/inputs/tcp.rb +101 -0
  393. data/spec/jar.rb +21 -0
  394. data/spec/outputs/csv.rb +266 -0
  395. data/spec/outputs/elasticsearch.rb +161 -0
  396. data/spec/outputs/elasticsearch_http.rb +240 -0
  397. data/spec/outputs/email.rb +173 -0
  398. data/spec/outputs/file.rb +82 -0
  399. data/spec/outputs/graphite.rb +236 -0
  400. data/spec/outputs/redis.rb +127 -0
  401. data/spec/speed.rb +20 -0
  402. data/spec/sqlite-test.rb +81 -0
  403. data/spec/support/LOGSTASH-733.rb +21 -0
  404. data/spec/support/LOGSTASH-820.rb +25 -0
  405. data/spec/support/akamai-grok.rb +26 -0
  406. data/spec/support/date-http.rb +17 -0
  407. data/spec/support/postwait1.rb +26 -0
  408. data/spec/support/pull375.rb +21 -0
  409. data/spec/test_utils.rb +125 -0
  410. data/spec/util/fieldeval_spec.rb +44 -0
  411. data/test/jenkins/config.xml.erb +74 -0
  412. data/test/jenkins/create-jobs.rb +23 -0
  413. data/test/jenkins/generatorjob.config.xml +66 -0
  414. data/tools/Gemfile +14 -0
  415. data/tools/Gemfile.jruby-1.9.lock +322 -0
  416. data/tools/Gemfile.rbx-2.1.lock +516 -0
  417. data/tools/Gemfile.ruby-1.9.1.lock +310 -0
  418. data/tools/Gemfile.ruby-2.0.0.lock +310 -0
  419. metadata +629 -0
@@ -0,0 +1,17 @@
1
+ logstash (1.1.10.dev~1) unstable; urgency=low
2
+
3
+ * Dev builds of master.
4
+
5
+ -- Jordan Sissel <jls@semicomplete.com> Tue, 2 Apr 2013 21:55:33 -0700
6
+
7
+ logstash (1.1.9~1) unstable; urgency=low
8
+
9
+ * Depend on openjdk. Don't start daemon by default.
10
+
11
+ -- Rudy Gevaert <Rudy.Gevaert@UGent.be> Wed, 27 Feb 2013 21:54:41 +0100
12
+
13
+ logstash (1.1.9) unstable; urgency=low
14
+
15
+ * Initial packaging release
16
+
17
+ -- Corey Quinn <corey@sequestered.net> Sun, 24 Feb 2013 18:24:41 -0800
@@ -0,0 +1 @@
1
+ 5
@@ -0,0 +1,16 @@
1
+ Source: logstash
2
+ Section: devel
3
+ Priority: extra
4
+ Maintainer: Corey Quinn <corey@sequestered.net>
5
+ Build-Depends: debhelper (>= 5)
6
+ Standards-Version: 3.7.2
7
+ Homepage: http://logstash.net
8
+
9
+ Package: logstash
10
+ Architecture: all
11
+ Depends: ${shlibs:Depends}, ${misc:Depends}, daemon, adduser, psmisc, default-jre
12
+ Description: tool for managing events and logs
13
+ logstash is a tool for managing events and logs. You can use it to collect logs,
14
+ parse them, and store them for later use (like, for searching). Speaking of
15
+ searching, logstash comes with a web interface for searching and drilling into
16
+ all of your logs.
@@ -0,0 +1,27 @@
1
+ This package was debianized by Corey Quinn <corey@sequestered.net>.
2
+
3
+ License:
4
+
5
+ This package is free software; you can redistribute it and/or modify
6
+ it under the terms of the GNU General Public License as published by
7
+ the Free Software Foundation; either version 2 of the License, or
8
+ (at your option) any later version.
9
+
10
+ This package is distributed in the hope that it will be useful,
11
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
12
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
+ GNU General Public License for more details.
14
+
15
+ You should have received a copy of the GNU General Public License
16
+ along with this package; if not, write to the Free Software
17
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
18
+
19
+ On Debian systems, the complete text of the GNU General
20
+ Public License can be found in `/usr/share/common-licenses/GPL'.
21
+
22
+ The Debian packaging is (C) 2013, Corey Quinn <corey@sequestered.net> and
23
+ is licensed under the GPL, see above.
24
+
25
+
26
+ # Please also look if there are files or directories which have a
27
+ # different copyright/license attached and list them here.
@@ -0,0 +1,19 @@
1
+ # moved from rules install
2
+ usr/share/logstash
3
+
4
+ # this is where LS_HOME is stored, so let's leave it as is for now
5
+ var/lib/logstash
6
+
7
+ # Need to create logstash's own directory to track the pid since the daemon is not run as root
8
+ # and doesn't have permission to write the pid to /var/run (which would be the preferred location).
9
+ var/run/logstash
10
+
11
+ # Store logstash log file in it's own directory since they can become rather large and in the future
12
+ # rotating logs can be easily added.
13
+ var/log/logstash
14
+
15
+ # Cache directory for the unpacked logstash.jar file.
16
+ var/cache/logstash
17
+
18
+ # Temp dir for java
19
+ var/logstash/
File without changes
@@ -0,0 +1,39 @@
1
+ # defaults for logstash
2
+
3
+ # Start logstash on boot?
4
+ START=no
5
+
6
+ # pulled in from the init script; makes things easier.
7
+ NAME=logstash
8
+
9
+ # location of java
10
+ JAVA=/usr/bin/java
11
+
12
+ # arguments to pass to java
13
+ LS_JAVA_OPTS="-Xmx256m -Djava.io.tmpdir=/var/lib/logstash/"
14
+
15
+ PIDFILE=/var/run/logstash.pid
16
+
17
+ # user id to be invoked as
18
+ LS_USER=logstash
19
+
20
+ # location of the logstas jar file
21
+ LS_JAR=/usr/share/logstash/logstash.jar
22
+
23
+ # logstash home location
24
+ LS_HOME=/var/lib/logstash
25
+
26
+ # logstash log directory
27
+ LOG_DIR=/var/log/logstash
28
+
29
+ # logstash log file
30
+ LOG_FILE=$LOG_DIR/$NAME.log
31
+
32
+ # logstash configuration directory
33
+ CONF_DIR=/etc/logstash/conf.d
34
+
35
+ # Open file limit
36
+ OPEN_FILES=2048
37
+
38
+ # Nice level
39
+ NICE=19
@@ -0,0 +1,201 @@
1
+ #!/bin/bash
2
+ #
3
+ # /etc/init.d/logstash -- startup script for LogStash.
4
+ #
5
+ ### BEGIN INIT INFO
6
+ # Provides: logstash
7
+ # Required-Start: $all
8
+ # Required-Stop: $all
9
+ # Default-Start: 2 3 4 5
10
+ # Default-Stop: 0 1 6
11
+ # Short-Description: Starts logstash
12
+ # Description: Starts logstash using start-stop-daemon
13
+ ### END INIT INFO
14
+
15
+ set -e
16
+
17
+ PATH=/bin:/usr/bin:/sbin:/usr/sbin
18
+ NAME=logstash
19
+ DESC="Logstash Daemon"
20
+ DEFAULT=/etc/default/$NAME
21
+
22
+ if [ `id -u` -ne 0 ]; then
23
+ echo "You need root privileges to run this script"
24
+ exit 1
25
+ fi
26
+
27
+ . /lib/lsb/init-functions
28
+
29
+ if [ -r /etc/default/rcS ]; then
30
+ . /etc/default/rcS
31
+ fi
32
+
33
+ # The following variables can be overwritten in $DEFAULT
34
+
35
+ # Run logstash as this user ID and group ID
36
+ LS_USER=logstash
37
+ LS_GROUP=logstash
38
+
39
+ JAVA=/usr/bin/java
40
+
41
+ # Directory where the logstash all in one jar lives
42
+ LS_HOME=/var/lib/logstash
43
+
44
+ # Additional Java OPTS
45
+ LS_JAVA_OPTS=" -Djava.io.tmpdir=/var/logstash/"
46
+
47
+ # logstash log directory
48
+ LOG_DIR=/var/log/logstash
49
+
50
+ # logstash configuration directory
51
+ CONF_DIR=/etc/logstash/conf.d
52
+
53
+ # logstash log file
54
+ LOG_FILE=$LOG_DIR/$NAME.log
55
+
56
+ # Open File limit
57
+ OPEN_FILES=2048
58
+
59
+
60
+
61
+ # Nice level
62
+ NICE=19
63
+
64
+ # End of variables that can be overwritten in $DEFAULT
65
+
66
+ # overwrite settings from default file
67
+ if [ -f "$DEFAULT" ]; then
68
+ . "$DEFAULT"
69
+ fi
70
+
71
+ # Define other required variables
72
+ PID_FILE=/var/run/$NAME.pid
73
+ DAEMON=$LS_JAR
74
+ DAEMON_OPTS="agent -f ${CONF_DIR} --log ${LOG_FILE}"
75
+
76
+ is_true() {
77
+ if [ "x$1" = "xtrue" -o "x$1" = "xyes" -o "x$1" = "x1" ] ; then
78
+ return 0
79
+ else
80
+ return 1
81
+ fi
82
+ }
83
+
84
+ # Check DAEMON exists
85
+ if ! test -e $DAEMON; then
86
+ log_failure_msg "Daemon $DAEMON doesn't exist"
87
+ exit 1
88
+ fi
89
+
90
+ case "$1" in
91
+ start)
92
+ if ! is_true "$START" ; then
93
+ echo "logstash not configured to start, please edit /etc/default/logstash to enable"
94
+ exit 0
95
+ fi
96
+
97
+ if [ -z "$JAVA" ]; then
98
+ log_failure_msg "no JDK found - $JAVA"
99
+ exit 1
100
+ fi
101
+
102
+ # Check if a config file exists
103
+ if ! test -e $CONF_DIR/*.conf; then
104
+ log_failure_msg "There aren't any configuration files in $CONF_DIR"
105
+ exit 1
106
+ fi
107
+
108
+ log_daemon_msg "Starting $DESC"
109
+
110
+ if start-stop-daemon --test --start --pidfile "$PID_FILE" \
111
+ --user "$LS_USER" --exec "$JAVA" \
112
+ >/dev/null; then
113
+ # Prepare environment
114
+ ulimit -n $OPEN_FILES
115
+
116
+ # Start Daemon
117
+ start-stop-daemon --start -b --user "$LS_USER" -c "$LS_USER":"$LS_GROUP" \
118
+ -d "$LS_HOME" --pidfile "$PID_FILE" --make-pidfile \
119
+ --exec "$JAVA" -- $LS_JAVA_OPTS -jar $DAEMON $DAEMON_OPTS
120
+
121
+ sleep 1
122
+
123
+ if start-stop-daemon --test --start --pidfile "$PID_FILE" \
124
+ --user "$LS_USER" --exec "$JAVA" \
125
+ >/dev/null; then
126
+
127
+ if [ -f "$PID_FILE" ]; then
128
+ rm -f "$PID_FILE"
129
+ fi
130
+
131
+ log_end_msg 1
132
+ else
133
+ log_end_msg 0
134
+ fi
135
+ else
136
+ log_progress_msg "(already running)"
137
+ log_end_msg 0
138
+ fi
139
+ ;;
140
+ stop)
141
+ log_daemon_msg "Stopping $DESC"
142
+
143
+ set +e
144
+
145
+ if [ -f "$PID_FILE" ]; then
146
+ start-stop-daemon --stop --pidfile "$PID_FILE" \
147
+ --user "$LS_USER" \
148
+ --retry=TERM/20/KILL/5 >/dev/null
149
+
150
+ if [ $? -eq 1 ]; then
151
+ log_progress_msg "$DESC is not running but pid file exists, cleaning up"
152
+ elif [ $? -eq 3 ]; then
153
+ PID="`cat $PID_FILE`"
154
+ log_failure_msg "Failed to stop $DESC (pid $PID)"
155
+ exit 1
156
+ fi
157
+
158
+ rm -f "$PID_FILE"
159
+ else
160
+ log_progress_msg "(not running)"
161
+ fi
162
+
163
+ log_end_msg 0
164
+ set -e
165
+ ;;
166
+ status)
167
+ set +e
168
+
169
+ start-stop-daemon --test --start --pidfile "$PID_FILE" \
170
+ --user "$LS_USER" --exec "$JAVA" \
171
+ >/dev/null 2>&1
172
+
173
+ if [ "$?" = "0" ]; then
174
+ if [ -f "$PID_FILE" ]; then
175
+ log_success_msg "$DESC is not running, but pid file exists."
176
+ exit 1
177
+ else
178
+ log_success_msg "$DESC is not running."
179
+ exit 3
180
+ fi
181
+ else
182
+ log_success_msg "$DESC is running with pid `cat $PID_FILE`"
183
+ fi
184
+
185
+ set -e
186
+ ;;
187
+ restart|force-reload)
188
+ if [ -f "$PID_FILE" ]; then
189
+ $0 stop
190
+ sleep 1
191
+ fi
192
+
193
+ $0 start
194
+ ;;
195
+ *)
196
+ log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
197
+ exit 1
198
+ ;;
199
+ esac
200
+
201
+ exit 0
@@ -0,0 +1 @@
1
+ logstash.jar usr/share/logstash
@@ -0,0 +1,9 @@
1
+ /var/log/logstash/logstash.log {
2
+ weekly
3
+ copytruncate
4
+ missingok
5
+ rotate 52
6
+ compress
7
+ delaycompress
8
+ notifempty
9
+ }
@@ -0,0 +1,68 @@
1
+ #!/bin/sh
2
+ # postinst script for jenkins
3
+ #
4
+ # see: dh_installdeb(1)
5
+
6
+ set -e
7
+
8
+ # summary of how this script can be called:
9
+ # * <postinst> `configure' <most-recently-configured-version>
10
+ # * <old-postinst> `abort-upgrade' <new version>
11
+ # * <conflictor's-postinst> `abort-remove' `in-favour' <package>
12
+ # <new-version>
13
+ # * <postinst> `abort-remove'
14
+ # * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
15
+ # <failed-install-package> <version> `removing'
16
+ # <conflicting-package> <version>
17
+ # for details, see http://www.debian.org/doc/debian-policy/ or
18
+ # the debian-policy package
19
+
20
+ LS_GROUP=logstash
21
+ LS_USER=logstash
22
+ case "$1" in
23
+ configure)
24
+
25
+ [ -r /etc/default/logstash ] && . /etc/default/logstash
26
+ : ${LS_USER:=logstash}
27
+
28
+ # Create logstash user if it doesn't exist.
29
+ # sometimes tools that users want logstash to run need a shell,
30
+ # so use /bin/bash.
31
+ if ! getent group "$LS_GROUP" > /dev/null 2>&1 ; then
32
+ addgroup --system "$LS_GROUP" --quiet
33
+ fi
34
+
35
+ if ! id $LS_USER > /dev/null 2>&1 ; then
36
+ adduser --system --home /var/lib/logstash --no-create-home \
37
+ --ingroup $LS_GROUP --disabled-password --shell /bin/bash $LS_USER
38
+ fi
39
+
40
+ # directories needed for logstash
41
+ # we don't do -R because it can take a long time on big installation
42
+ chown $LS_USER:adm /var/lib/logstash /var/log/logstash
43
+ # we don't do "chmod 750" so that the user can choose the pemission for g and o on their own
44
+ chmod u+rwx /var/lib/logstash /var/log/logstash
45
+
46
+ # make sure logstash can delete everything in /var/cache/logstash to
47
+ # re-explode jar.
48
+
49
+ chown -R $LS_USER:adm /var/cache/logstash /var/run/logstash
50
+ chmod -R 750 /var/cache/logstash /var/run/logstash
51
+ ;;
52
+
53
+ abort-upgrade|abort-remove|abort-deconfigure)
54
+ ;;
55
+
56
+ *)
57
+ echo "postinst called with unknown argument \`$1'" >&2
58
+ exit 1
59
+ ;;
60
+ esac
61
+
62
+ # dh_installdeb will replace this with shell code automatically
63
+ # generated by other debhelper scripts.
64
+
65
+ #DEBHELPER#
66
+
67
+ exit 0
68
+
@@ -0,0 +1,23 @@
1
+ #!/bin/sh
2
+
3
+ set -e
4
+
5
+ case "$1" in
6
+ purge)
7
+ userdel logstash || true
8
+ rm -rf /var/lib/logstash /var/log/logstash \
9
+ /var/run/logstash /var/cache/logstash
10
+ ;;
11
+
12
+ remove|upgrade|failed-upgrade|abort-install|abort-upgrade|disappear)
13
+ ;;
14
+
15
+ *)
16
+ echo "postrm called with unknown argument \`$1'" >&2
17
+ exit 1
18
+ ;;
19
+ esac
20
+
21
+ #DEBHELPER#
22
+
23
+ exit 0
@@ -0,0 +1,59 @@
1
+ .\" Hey, EMACS: -*- nroff -*-
2
+ .\" First parameter, NAME, should be all caps
3
+ .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection
4
+ .\" other parameters are allowed: see man(7), man(1)
5
+ .TH LOGSTASH SECTION "April 25, 2008"
6
+ .\" Please adjust this date whenever revising the manpage.
7
+ .\"
8
+ .\" Some roff macros, for reference:
9
+ .\" .nh disable hyphenation
10
+ .\" .hy enable hyphenation
11
+ .\" .ad l left justify
12
+ .\" .ad b justify to both left and right margins
13
+ .\" .nf disable filling
14
+ .\" .fi enable filling
15
+ .\" .br insert line break
16
+ .\" .sp <n> insert n+1 empty lines
17
+ .\" for manpage-specific macros, see man(7)
18
+ .SH NAME
19
+ logstash \- program to do something
20
+ .SH SYNOPSIS
21
+ .B logstash
22
+ .RI [ options ] " files" ...
23
+ .br
24
+ .B bar
25
+ .RI [ options ] " files" ...
26
+ .SH DESCRIPTION
27
+ This manual page documents briefly the
28
+ .B logstash
29
+ and
30
+ .B bar
31
+ commands.
32
+ .PP
33
+ .\" TeX users may be more comfortable with the \fB<whatever>\fP and
34
+ .\" \fI<whatever>\fP escape sequences to invode bold face and italics,
35
+ .\" respectively.
36
+ \fBlogstash\fP is a program that...
37
+ .SH OPTIONS
38
+ These programs follow the usual GNU command line syntax, with long
39
+ options starting with two dashes (`-').
40
+ A summary of options is included below.
41
+ For a complete description, see the Info files.
42
+ .TP
43
+ .B \-h, \-\-help
44
+ Show summary of options.
45
+ .TP
46
+ .B \-v, \-\-version
47
+ Show version of program.
48
+ .SH SEE ALSO
49
+ .BR bar (1),
50
+ .BR baz (1).
51
+ .br
52
+ The programs are documented fully by
53
+ .IR "The Rise and Fall of a Fooish Bar" ,
54
+ available via the Info system.
55
+ .SH AUTHOR
56
+ logstash was written by <upstream author>.
57
+ .PP
58
+ This manual page was written by Kohsuke Kawaguchi <kk@kohsuke.org>,
59
+ for the Debian project (but may be used by others).