influxdb-plugin-fluent 1.0.0.pre.183

data/bin/influxdb-restart.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+#
+# The MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+set -e
+
+DEFAULT_DOCKER_REGISTRY="quay.io/influxdb/"
+DOCKER_REGISTRY="${DOCKER_REGISTRY:-$DEFAULT_DOCKER_REGISTRY}"
+
+DEFAULT_INFLUXDB_V2_REPOSITORY="influxdb"
+DEFAULT_INFLUXDB_V2_VERSION="2.0.0-beta"
+INFLUXDB_V2_REPOSITORY="${INFLUXDB_V2_REPOSITORY:-$DEFAULT_INFLUXDB_V2_REPOSITORY}"
+INFLUXDB_V2_VERSION="${INFLUXDB_V2_VERSION:-$DEFAULT_INFLUXDB_V2_VERSION}"
+INFLUXDB_V2_IMAGE=${DOCKER_REGISTRY}${INFLUXDB_V2_REPOSITORY}:${INFLUXDB_V2_VERSION}
+
+SCRIPT_PATH="$( cd "$(dirname "$0")" ; pwd -P )"
+
+docker kill influxdb_v2 || true
+docker rm influxdb_v2 || true
+docker network rm influx_network || true
+docker network create -d bridge influx_network --subnet 192.168.0.0/24 --gateway 192.168.0.1
+
+#
+# InfluxDB 2.0
+#
+echo
+echo "Restarting InfluxDB 2.0 [${INFLUXDB_V2_IMAGE}] ... "
+echo
+
+docker pull "${INFLUXDB_V2_IMAGE}" || true
+docker run \
+       --detach \
+       --name influxdb_v2 \
+       --network influx_network \
+       --publish 9999:9999 \
+       "${INFLUXDB_V2_IMAGE}"
+
+#
+# Post onBoarding request to InfluxDB 2
+#
+"${SCRIPT_PATH}"/influxdb-onboarding.sh

data/examples/README.md

@@ -0,0 +1,190 @@
+# InfluxDB 2 + Fluentd
+
+InfluxDB 2 and Fluentd together are able to collect large amount of logs and transform it to useful metrics. 
+InfluxDB 2 provide a solution for realtime analysis and alerting over collected metrics.
+
+<img src="architecture.png" height="400px">
+
+## Introduction
+
+[Fluentd](https://www.fluentd.org/architecture) is an open source data collector, which lets you unify the data collection and consumption for a better use and understanding of data.
+
+[InfluxDB](https://www.influxdata.com) open source time series database, purpose-built by InfluxData for monitoring metrics and events, provides real-time visibility into stacks, sensors, and systems. 
+
+## Demo
+
+The following demo show how to analyze logs (Apache Access Log) from dockerized environment.
+
+> The steps from 1 to 6 could be skipped if you use a script:
+>
+> [`run-example.sh`](run-example.sh)
+### Prerequisites
+
+- Docker installed on your computer
+
+### Step 1 — Create Docker Network
+
+Create bridge network that allows smoothly communication between containers:
+
+```bash
+docker network create -d bridge influx_network --subnet 192.168.0.0/24 --gateway 192.168.0.1
+```
+
+### Step 2 — Start InfluxDB
+
+Start latest `InfluxDB 2`:
+
+```bash
+docker run \
+       --detach \
+       --name influxdb_v2 \
+       --network influx_network \
+       --publish 9999:9999 \
+       quay.io/influxdb/influxdb:2.0.0-beta
+```
+
+Create default organization, user and bucket:
+```bash
+curl -i -X POST http://localhost:9999/api/v2/setup -H 'accept: application/json' \
+    -d '{
+            "username": "my-user",
+            "password": "my-password",
+            "org": "my-org",
+            "bucket": "my-bucket",
+            "token": "my-token"
+        }'
+```
+
+### Step 3 — Prepare Fluentd Docker
+We have to prepare a docker image that will contains Fluentd with configured [InfluxDB 2 output plugin](https://github.com/bonitoo-io/influxdb-plugin-fluent/).
+
+Fluentd is configured to parse incoming events with tag `httpd.access` by regexp: `/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/` to structured event with: `time`, `host`, `user`, `method`, `code` and `size`.
+These structured events are routed to InfluxDB 2 output plugin. 
+#### Required files:
+
+##### Dockerfile
+
+```dockerfile
+FROM fluent/fluentd:edge-debian
+
+USER root
+
+RUN fluent-gem install influxdb-plugin-fluent
+
+COPY ./fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+
+USER fluent
+```
+
+##### fluent.conf
+
+```xml
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+<source>
+  @type monitor_agent
+  bind 0.0.0.0
+  port 24220
+</source>
+<filter httpd.access>
+  @type parser
+  key_name log
+  <parse>
+    @type regexp
+    expression /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
+    time_format %d/%b/%Y:%H:%M:%S %z
+  </parse>
+</filter>
+<match httpd.access>
+  @type copy
+  <store>
+    @type influxdb2
+    url http://influxdb_v2:9999
+    token my-token
+    bucket my-bucket
+    org my-org
+    use_ssl false
+    time_precision s
+    tag_keys ["method", "host", "path"]
+    <buffer tag>
+      @type memory
+      flush_interval 5
+    </buffer>
+  </store>
+</match>
+```
+Build image: 
+
+```bash
+docker build -t fluentd_influx .
+```
+### Step 4 — Start Fluentd image
+
+```bash
+docker run \
+       --detach \
+       --name fluentd_influx \
+       --network influx_network \
+       --publish 24224:24224 \
+       --publish 24220:24220 \
+       fluentd_influx
+```
+
+### Step 5 — Start Apache HTTP Server
+
+Docker includes multiple logging mechanisms to help you get information from running containers and services.
+
+We will use [Fluentd](https://docs.docker.com/config/containers/logging/fluentd/) logging driver with configured tag as `httpd.access`:
+ 
+```bash
+docker run \
+       --detach \
+       --name web \
+       --network influx_network \
+       --publish 8080:80 \
+       --log-driver fluentd \
+       --log-opt tag=httpd.access \
+       httpd
+```
+
+### Step 6 — Generate httpd Access Logs
+
+Generate some access logs by curl:
+```bash
+curl http://localhost:8080/
+curl http://localhost:8080/not_exists
+```
+
+### Step 7 — Import Dashboard
+
+Open [InfluxDB](http://localhost:9999) and import dashboard [web_app_access.json](influxdb/web_app_access.json) by following steps:
+
+```
+username: my-user
+password: my-password
+```
+
+1. Click the **Dashboards** icon in the navigation bar.
+1. Click the **Create Dashboard** menu in the upper right and select **Import Dashboard**.
+1. Select **Upload File** to drag-and-drop or select a **web_app_access.json**.
+1. Click **Import JSON** as Dashboard.
+
+The imported dashboard should look like this:
+
+<img src="dashboard.png" height="400px">
+
+ 
+## Conclusion
+
+Analyze Apache Access Log is just one way how to use a power of InfluxDB and Fluentd. 
+There are other things you could do with InfluxDB and Fluentd such as: [Monitoring and alerting](https://v2.docs.influxdata.com/v2.0/monitor-alert/#manage-your-monitoring-and-alerting-pipeline). 
+
+## Links
+
+- https://www.digitalocean.com/community/tutorials/how-to-centralize-your-docker-logs-with-fluentd-and-elasticsearch-on-ubuntu-16-04
+- https://stackoverflow.com/questions/58563760/docker-compose-pulls-an-two-images-an-app-and-fluentd-but-no-logs-are-sent-to-s
+- https://docs.fluentd.org/v/0.12/container-deployment/docker-compose

data/examples/ab/load.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# The MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+
+apk add parallel
+cat /tmp/urls.txt | parallel "watch -n 5 ab -n 1000 -c 10 {}"

data/examples/ab/urls.txt

@@ -0,0 +1,4 @@
+http://web:80/foo/bar/
+http://web:80/
+http://web:80/foo.html
+http://web:80/parallelfoo/bar/?v=1

data/examples/fluentd/Dockerfile

@@ -0,0 +1,12 @@
+FROM fluent/fluentd:edge-debian
+
+USER root
+
+COPY ./influxdb-plugin-fluent.gem /fluentd/plugins
+
+RUN fluent-gem install /fluentd/plugins/influxdb-plugin-fluent.gem
+
+COPY ./fluent.conf /fluentd/etc/
+COPY entrypoint.sh /bin/
+
+USER fluent

data/examples/fluentd/entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+#source vars if file exists
+DEFAULT=/etc/default/fluentd
+
+if [ -r $DEFAULT ]; then
+    set -o allexport
+    . $DEFAULT
+    set +o allexport
+fi
+
+# If the user has supplied only arguments append them to `fluentd` command
+if [ "${1#-}" != "$1" ]; then
+    set -- fluentd "$@"
+fi
+
+# If user does not supply config file or plugins, use the default
+if [ "$1" = "fluentd" ]; then
+    if ! echo $@ | grep ' \-c' ; then
+       set -- "$@" -c /fluentd/etc/${FLUENTD_CONF}
+    fi
+
+    if ! echo $@ | grep ' \-p' ; then
+       set -- "$@" -p /fluentd/plugins
+    fi
+fi
+
+exec "$@"

data/examples/fluentd/fluent.conf

@@ -0,0 +1,40 @@
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+<source>
+  @type monitor_agent
+  bind 0.0.0.0
+  port 24220
+</source>
+<filter httpd.access>
+  @type parser
+  key_name log
+  <parse>
+    @type regexp
+    expression /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)$/
+    time_format %d/%b/%Y:%H:%M:%S %z
+  </parse>
+</filter>
+<match httpd.access>
+  @type copy
+  <store>
+    @type influxdb2
+    # @log_level trace
+    url http://influxdb_v2:9999
+    token my-token
+    bucket my-bucket
+    org my-org
+    use_ssl false
+    time_precision s
+    tag_keys ["method", "host", "path"]
+    <buffer tag>
+      @type memory
+      flush_interval 5
+    </buffer>
+  </store>
+#  <store>
+#    @type stdout
+#  </store>
+</match>

data/examples/influxdb/web_app_access.json

@@ -0,0 +1,365 @@
+{
+ "meta": {
+  "version": "1",
+  "type": "dashboard",
+  "name": "Web App Access-Template",
+  "description": "template created from dashboard: Web App Access"
+ },
+ "content": {
+  "data": {
+   "type": "dashboard",
+   "attributes": {
+    "name": "Web App Access",
+    "description": ""
+   },
+   "relationships": {
+    "label": {
+     "data": []
+    },
+    "cell": {
+     "data": [
+      {
+       "type": "cell",
+       "id": "051559b2aa2a2000"
+      },
+      {
+       "type": "cell",
+       "id": "051559b2ae2a2000"
+      },
+      {
+       "type": "cell",
+       "id": "051559b2af6a2000"
+      },
+      {
+       "type": "cell",
+       "id": "05155aef3daa2000"
+      }
+     ]
+    },
+    "variable": {
+     "data": []
+    }
+   }
+  },
+  "included": [
+   {
+    "id": "051559b2aa2a2000",
+    "type": "cell",
+    "attributes": {
+     "x": 0,
+     "y": 0,
+     "w": 4,
+     "h": 4
+    },
+    "relationships": {
+     "view": {
+      "data": {
+       "type": "view",
+       "id": "051559b2aa2a2000"
+      }
+     }
+    }
+   },
+   {
+    "id": "051559b2ae2a2000",
+    "type": "cell",
+    "attributes": {
+     "x": 0,
+     "y": 4,
+     "w": 12,
+     "h": 4
+    },
+    "relationships": {
+     "view": {
+      "data": {
+       "type": "view",
+       "id": "051559b2ae2a2000"
+      }
+     }
+    }
+   },
+   {
+    "id": "051559b2af6a2000",
+    "type": "cell",
+    "attributes": {
+     "x": 4,
+     "y": 0,
+     "w": 4,
+     "h": 4
+    },
+    "relationships": {
+     "view": {
+      "data": {
+       "type": "view",
+       "id": "051559b2af6a2000"
+      }
+     }
+    }
+   },
+   {
+    "id": "05155aef3daa2000",
+    "type": "cell",
+    "attributes": {
+     "x": 8,
+     "y": 0,
+     "w": 4,
+     "h": 4
+    },
+    "relationships": {
+     "view": {
+      "data": {
+       "type": "view",
+       "id": "05155aef3daa2000"
+      }
+     }
+    }
+   },
+   {
+    "type": "view",
+    "id": "051559b2aa2a2000",
+    "attributes": {
+     "name": "Requests Count",
+     "properties": {
+      "shape": "chronograf-v2",
+      "type": "single-stat",
+      "queries": [
+       {
+        "text": "from(bucket: \"my-bucket\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r._measurement == \"httpd.access\")\n  |> filter(fn: (r) => r._field == \"size\")\n  |> drop(columns: [\"host\", \"method\", \"path\"])\n  |> count()",
+        "editMode": "advanced",
+        "name": "",
+        "builderConfig": {
+         "buckets": [],
+         "tags": [
+          {
+           "key": "_measurement",
+           "values": []
+          }
+         ],
+         "functions": [],
+         "aggregateWindow": {
+          "period": "auto"
+         }
+        }
+       }
+      ],
+      "prefix": "",
+      "suffix": "",
+      "colors": [
+       {
+        "id": "base",
+        "type": "text",
+        "hex": "#00C9FF",
+        "name": "laser",
+        "value": 0
+       }
+      ],
+      "decimalPlaces": {
+       "isEnforced": true,
+       "digits": 2
+      },
+      "note": "",
+      "showNoteWhenEmpty": false
+     }
+    }
+   },
+   {
+    "type": "view",
+    "id": "051559b2ae2a2000",
+    "attributes": {
+     "name": "Requests",
+     "properties": {
+      "shape": "chronograf-v2",
+      "queries": [
+       {
+        "text": "from(bucket: \"my-bucket\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r._measurement == \"httpd.access\")\n  |> filter(fn: (r) => r._field == \"code\")\n  |> drop(columns: [\"method\", \"host\"])\n  |> aggregateWindow(every: 1m, fn: count)",
+        "editMode": "advanced",
+        "name": "",
+        "builderConfig": {
+         "buckets": [],
+         "tags": [
+          {
+           "key": "_measurement",
+           "values": []
+          }
+         ],
+         "functions": [],
+         "aggregateWindow": {
+          "period": "auto"
+         }
+        }
+       }
+      ],
+      "axes": {
+       "x": {
+        "bounds": [
+         "",
+         ""
+        ],
+        "label": "",
+        "prefix": "",
+        "suffix": "",
+        "base": "10",
+        "scale": "linear"
+       },
+       "y": {
+        "bounds": [
+         "",
+         ""
+        ],
+        "label": "",
+        "prefix": "",
+        "suffix": "",
+        "base": "10",
+        "scale": "linear"
+       }
+      },
+      "type": "xy",
+      "legend": {},
+      "geom": "line",
+      "colors": [
+       {
+        "id": "56d087db-1d89-453c-8787-d5b5368b608f",
+        "type": "scale",
+        "hex": "#31C0F6",
+        "name": "Nineteen Eighty Four",
+        "value": 0
+       },
+       {
+        "id": "e5d0bd7c-cded-4ed2-aa55-94e920105d6e",
+        "type": "scale",
+        "hex": "#A500A5",
+        "name": "Nineteen Eighty Four",
+        "value": 0
+       },
+       {
+        "id": "e5f1b958-099c-4d84-8e78-8a460b8193e5",
+        "type": "scale",
+        "hex": "#FF7E27",
+        "name": "Nineteen Eighty Four",
+        "value": 0
+       }
+      ],
+      "note": "",
+      "showNoteWhenEmpty": false,
+      "xColumn": "_time",
+      "yColumn": "_value",
+      "shadeBelow": true,
+      "position": "overlaid",
+      "timeFormat": ""
+     }
+    }
+   },
+   {
+    "type": "view",
+    "id": "051559b2af6a2000",
+    "attributes": {
+     "name": "Request Size",
+     "properties": {
+      "shape": "chronograf-v2",
+      "type": "single-stat",
+      "queries": [
+       {
+        "text": "from(bucket: \"my-bucket\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r._measurement == \"httpd.access\")\n  |> filter(fn: (r) => r.method == \"GET\")\n  |> filter(fn: (r) => r._field == \"size\")\n  |> drop(columns: [\"path\"])\n  |> toInt()\n  |> sum(column: \"_value\")\n  |> map(fn: (r) => ({\n      r with\n      _value: r._value / 1024\n    })\n  )",
+        "editMode": "advanced",
+        "name": "",
+        "builderConfig": {
+         "buckets": [],
+         "tags": [
+          {
+           "key": "_measurement",
+           "values": []
+          }
+         ],
+         "functions": [],
+         "aggregateWindow": {
+          "period": "auto"
+         }
+        }
+       }
+      ],
+      "prefix": "",
+      "suffix": " KiB",
+      "colors": [
+       {
+        "id": "base",
+        "type": "text",
+        "hex": "#F48D38",
+        "name": "tiger",
+        "value": 0
+       }
+      ],
+      "decimalPlaces": {
+       "isEnforced": true,
+       "digits": 2
+      },
+      "note": "",
+      "showNoteWhenEmpty": false
+     }
+    }
+   },
+   {
+    "type": "view",
+    "id": "05155aef3daa2000",
+    "attributes": {
+     "name": "Success rate",
+     "properties": {
+      "shape": "chronograf-v2",
+      "type": "single-stat",
+      "queries": [
+       {
+        "text": "from(bucket: \"my-bucket\")\n  |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n  |> filter(fn: (r) => r._measurement == \"httpd.access\")\n  |> filter(fn: (r) => r._field == \"code\")\n  |> drop(columns: [\"path\", \"method\", \"host\"])\n  |> toInt()\n  |> map(fn: (r) => ({   \n        _time: r._start,\n        _success:\n          if r._value < 300 then 1\n          else 0,\n        _failure:\n          if r._value >= 300 then 1\n          else 0\n       })\n    )\n  |> cumulativeSum(columns: [\"_failure\", \"_success\"])\n  |> drop(columns: [\"_time\"])\n  |> map(fn: (r) => ({\n    r with\n    _value: (float(v: r._success) / (float(v: r._success) + float(v: r._failure))) * 100.0\n  })\n)\n  |> last()",
+        "editMode": "advanced",
+        "name": "",
+        "builderConfig": {
+         "buckets": [],
+         "tags": [
+          {
+           "key": "_measurement",
+           "values": []
+          }
+         ],
+         "functions": [],
+         "aggregateWindow": {
+          "period": "auto"
+         }
+        }
+       }
+      ],
+      "prefix": "",
+      "suffix": "%",
+      "colors": [
+       {
+        "id": "base",
+        "type": "text",
+        "hex": "#00C9FF",
+        "name": "laser",
+        "value": 0
+       },
+       {
+        "id": "5e50c5ba-51cc-4139-870f-663050d6e79d",
+        "type": "text",
+        "hex": "#F48D38",
+        "name": "tiger",
+        "value": 80
+       },
+       {
+        "id": "305d340f-09e5-4137-b88b-0a6599209dc0",
+        "type": "text",
+        "hex": "#7CE490",
+        "name": "honeydew",
+        "value": 90
+       }
+      ],
+      "decimalPlaces": {
+       "isEnforced": true,
+       "digits": 2
+      },
+      "note": "",
+      "showNoteWhenEmpty": false
+     }
+    }
+   }
+  ]
+ },
+ "labels": []
+}