@whitewater-guide/gorge 2.4.0-beta.8 → 3.0.0-beta.4

package/README.md

@@ -13,6 +13,7 @@ Harvested data is stored in database and can be queried later.
   - [Launching](#launching)
   - [Working with API](#working-with-api)
   - [Available scripts](#available-scripts)
+  - [Health notifications](#health-notifications)
   - [Other](#other)
 - [Development](#development)
   - [Inside container](#inside-container)
@@ -45,7 +46,7 @@ You can find the list of our data sources and their statuses [here](scripts/READ
 
 ## Usage
 
-Gorge is distributed as a ~50Mb [docker image](https://github.com/whitewater-guide/gorge/pkgs/container/gorge) with two binary files:
+Gorge is distributed as a ~130Mb [docker image](https://github.com/whitewater-guide/gorge/pkgs/container/gorge) with two binary files:
 
 - `gorge-server` (_entrypoint_) - web server with REST API
 - `gorge-cli` - command-line client for this server. Since image is distroless, use `docker exec gorge gorge-cli` to call it
@@ -84,24 +85,28 @@ command:
 Here is the list of available flags:
 
 ```
---cache string             Either 'inmemory' or 'redis' (default "redis")
---db string                Either 'inmemory' or 'postgres' (default "postgres")
---db-chunk-size int        Measurements will be saved to db in chunks of this size. When set to 0, they will be saved in one chunk, which can cause errors
---debug                    Enables debug mode, sets log level to debug
---endpoint string          Endpoint path (default "/")
---http-proxy string        HTTP client proxy (for example, you can use mitm for local development)
---http-timeout int         Request timeout in seconds (default 60)
---http-user-agent string   User agent for requests sent from scripts. Leave empty to use fake browser agent (default "whitewater.guide robot")
---http-without-tls         Disable TLS for some gauges
---log-format string        Set this to 'json' to output log in json (default "json")
---log-level string         Log level. Leave empty to discard logs (default "info")
---pg-db string             Postgres database (default "postgres")
---pg-host string           Postgres host (default "db")
---pg-password string       Postgres password [env POSTGRES_PASSWORD]
---pg-user string           Postgres user (default "postgres")
---port string              Port (default "7080")
---redis-host string        Redis host (default "redis")
---redis-port string        Redis port (default "6379")
+--cache string                   either 'inmemory' or 'redis' (default "redis")
+--db string                      either 'inmemory' or 'postgres' (default "postgres")
+--db-chunk-size int              measurements will be saved to db in chunks of this size. When set to 0, they will be saved in one chunk, which can cause errors
+--debug                          enables debug mode, sets log level to debug
+--endpoint string                endpoint path (default "/")
+--hooks-health-cron string       cron expression for running health notifier (default "0 0 * * *")
+--hooks-health-headers strings   headers to set on request, in 'Header: Value' format, similar to curl  (default [])
+--hooks-health-threshold int     hours required to pass since last successful execution to consider job unhealthy (default 48)
+--hooks-health-url string        external endpoint to call with list of unhealthy jobs
+--http-proxy string              HTTP client proxy (for example, you can use mitm for local development)
+--http-timeout int               Request timeout in seconds (default 60)
+--http-user-agent string         User agent for requests sent from scripts. Leave empty to use fake browser agent (default "whitewater.guide robot")
+--http-without-tls               Disable TLS for some gauges
+--log-format string              set this to 'json' to output log in json (default "json")
+--log-level string               log level. Leave empty to discard logs (default "info")
+--pg-db string                   postgres database (default "postgres")
+--pg-host string                 postgres host (default "db")
+--pg-password string             postgres password [env POSTGRES_PASSWORD]
+--pg-user string                 postgres user (default "postgres")
+--port string                    port (default "7080")
+--redis-host string              redis host (default "redis")
+--redis-port string              redis port (default "6379")
 ```
 
 Gorge uses database to store harvested measurements and scheduled jobs. It comes with postgres and sqlite drivers. Gorge will initialize all the required tables. Check out sql migration file if you're curious about db schema.
@@ -222,11 +227,11 @@ Below is the list of endpoints exposed by gorge server. You can use `request.htt
       },
       "status": {
         // information about running job
-        "success": true, // whether latest execution was successful
-        "timestamp": "2020-02-25T17:44:00Z", // latest execution timestamp
+        "lastRun": "2020-02-25T17:44:00Z", // latest execution timestamp
+        "lastSuccess": "2020-02-25T17:44:00Z", // latest successful (>0 measurements harvested) execution timestamp (optional)
         "count": 10, // number of measurements harvested during latest execution
-        "next": "2020-02-25T17:46:00Z", // next execution timestamp
-        "error": "somethin went wrong" // latest execution error, omitted when success = true
+        "nextRun": "2020-02-25T17:46:00Z", // next execution timestamp
+        "error": "somethin went wrong" // latest execution error (optional)
       }
     }
   ]
@@ -267,10 +272,10 @@ Below is the list of endpoints exposed by gorge server. You can use `request.htt
   [
     {
     "010802": {
-      "success": false,
-      "timestamp": "2020-02-24T18:00:00Z",
-      "count": 0,
-      "next": "2020-02-25T18:00:00Z"
+      "lastRun": "2020-02-24T18:00:00Z",
+      "lastSuccess": "2020-02-24T18:00:00Z",
+      "count": 10,
+      "nextRun": "2020-02-25T18:00:00Z"
     }
   ]
   ```
@@ -350,6 +355,48 @@ Below is the list of endpoints exposed by gorge server. You can use `request.htt
 
 List of available scripts is [here](scripts/README.md)
 
+### Health notifications
+
+Gorge can call your webhooks when some of the running scripts haven't harvested any data for a period of time.
+
+To configure healthcheck, use `--health--xxx` cli arguments. For example:
+
+```yml
+command:
+  [
+    
+    '--hooks-health-cron',
+    '0 0 * * *', # check health every midnight
+
+    '--hooks-health-threshold',
+    '48', # scripts that haven't harvested anything within last 48 hours are considered unhealthy
+
+    '--hooks-health-url',
+    'http://host.docker.internal:3333/gorge/health', # so POST request will be made to this endpoint
+
+    '--hooks-health-headers',
+    'x-api-key: __test_gorge_health_key__', # multiple headers can be set on this request
+  ]
+```
+
+Example of this POST request payload:
+
+```json
+[
+    {
+        "id": "2f915d20-ffe6-11e8-8919-9f370230d1ae",
+        "script": "chile",
+        "lastRun": "2021-12-13T07:57:59Z"
+    },
+    {
+        "id": "e3c0c89a-7c72-11e9-8abd-cfc3ab2b843d",
+        "script": "quebec",
+        "lastRun": "2021-12-13T07:57:00Z",
+        "lastSuccess": "2021-12-10T09:22:00Z"
+    }
+]
+```
+
 ### Other
 
 There're Typescript type definitions for the API available on [NPM](https://www.npmjs.com/package/@whitewater-guide/gorge)
@@ -416,7 +463,6 @@ Here are some recommendations for writing scripts for new sources
 
 ## TODO
 
-- Notify when some script seem to be broken
 - Virtual gauges
   - Statuses
   - What happens when one component is broken?

package/index.d.ts

@@ -24,11 +24,11 @@ export interface Measurement {
     flow: number | null;
 }
 export interface Status {
-    success: boolean;
-    timestamp: string;
+    lastRun: string;
     error?: string;
     count: number;
-    next?: string;
+    lastSuccess?: string;
+    nextRun?: string;
 }
 export interface JobDescription {
     id: string;
@@ -38,6 +38,12 @@ export interface JobDescription {
     options: {[key: string]: any} | null;
     status?: Status;
 }
+export interface UnhealthyJob {
+    id: string;
+    script: string;
+    lastRun: string;
+    lastSuccess?: string;
+}
 export interface ScriptDescriptor {
     name: string;
     description: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@whitewater-guide/gorge",
-  "version": "2.4.0-beta.8",
+  "version": "3.0.0-beta.4",
   "description": "Hydrological data harvester",
   "main": "",
   "types": "index",