@socialgouv/matomo-postgres 1.5.0 → 1.5.2-beta.1

package/README.md

@@ -4,8 +4,6 @@
 
 Extract matomo data from [`Live.getLastVisitsDetails`](https://developer.matomo.org/api-reference/reporting-api) API and push events and visits informations to Postgres.
 
-Use [pg_partman](https://github.com/pgpartman/pg_partman) to partition data by month.
-
 ## Usage
 
 Create the [initial table](./initial.sql) database table then run the following job with correct environment variables.
@@ -16,16 +14,16 @@ npx @socialgouv/matomo-postgres
 
 ### Environment variables Deployment
 
-| name              | value                                          |
-| ----------------- | ---------------------------------------------- |
-| MATOMO_KEY\*      | matomo api token                               |
-| MATOMO_SITE\*     | matomo site id                                 |
-| MATOMO_URL\*      | matomo url                                     |
-| PGDATABASE\*      | Postgres connection string                     |
-| DESTINATION_TABLE | `matomo`                                       |
-| STARTDATE         | default to today()                             |
-| RESULTPERPAGE     | matomo pagination : `100`                      |
-| OFFSET            | default days to check in the past; default = 3 |
+| name              | value                                                    |
+| ----------------- | -------------------------------------------------------- |
+| MATOMO_KEY\*      | matomo api token                                         |
+| MATOMO_SITE\*     | matomo site id                                           |
+| MATOMO_URL\*      | matomo url                                               |
+| PGDATABASE\*      | Postgres connection string                               |
+| DESTINATION_TABLE | `matomo`                                                 |
+| STARTDATE         | default to today()                                       |
+| RESULTPERPAGE     | matomo pagination (defaults to 500)                      |
+| INITIAL_OFFSET    | How many days to fetch on initialisation (defaults to 3) |
 
 ## Dev
 
@@ -43,3 +41,6 @@ yarn start
 
 Use `yarn test -u` to update the snapshots
 
+## Database migrations
+
+`yarn migrate` is run on each `yarn start` with Kysely migrations at [./src/migrations](./src/migrations/)

package/bin/index.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-const run = require("../src/index");
+const { default: run } = require("../dist/index");
 
 if (require.main === module) {
   const date =

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@socialgouv/matomo-postgres",
   "description": "Extract visitor events from Matomo API and push to Postgres",
-  "version": "1.5.0",
+  "version": "1.5.2-beta.1",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",
-  "main": "src/index.js",
+  "main": "dist/index.js",
   "preferGlobal": true,
   "publishConfig": {
     "access": "public"
@@ -12,25 +12,37 @@
   "bin": {
     "matomo-postgres": "./bin/index.js"
   },
+  "files": [
+    "bin",
+    "dist"
+  ],
   "scripts": {
-    "start": "node ./src/index.js",
+    "start": "yarn migrate && node ./dist/index.js",
+    "build": "tsc",
+    "prepublish": "yarn build",
+    "migrate": "node ./dist/migrate-latest.js",
     "test": "jest --verbose"
   },
   "prettier": {
     "printWidth": 120
   },
   "dependencies": {
-    "date-fns": "^2.23.0",
-    "debug": "^4.3.3",
-    "dotenv": "^10.0.0",
-    "p-all": "^3.0.0",
-    "pg": "^8.7.1",
+    "date-fns": "^2.29.3",
+    "debug": "^4.3.4",
+    "dotenv": "^16.0.3",
+    "kysely": "^0.23.4",
+    "p-all": "^3",
+    "pg": "^8.9.0",
     "piwik-client": "^0.2.2"
   },
   "devDependencies": {
-    "@types/jest": "^27.0.3",
-    "@types/pg": "^8.6.1",
-    "jest": "^27.1.1",
-    "typescript": "^4.4.4"
+    "@types/debug": "^4.1.7",
+    "@types/jest": "^29.4.0",
+    "@types/node": "^18.14.4",
+    "@types/pg": "^8.6.6",
+    "jest": "^29.4.3",
+    "ts-jest": "^29.0.5",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.5"
   }
 }

package/.github/workflows/release.yml

@@ -1,33 +0,0 @@
-name: Release
-
-on:
-  workflow_dispatch:
-  push:
-    branches: [master, alpha, beta, next]
-
-jobs:
-  release:
-    name: Release
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
-        with:
-          semantic_version: 17
-          extra_plugins: |
-            @semantic-release/changelog@5.0.1
-            @semantic-release/git@9.0.0
-
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GIT_AUTHOR_EMAIL: ${{ secrets.SOCIALGROOVYBOT_EMAIL }}
-          GIT_AUTHOR_NAME: ${{ secrets.SOCIALGROOVYBOT_NAME }}
-          GIT_COMMITTER_EMAIL: ${{ secrets.SOCIALGROOVYBOT_EMAIL }}
-          GIT_COMMITTER_NAME: ${{ secrets.SOCIALGROOVYBOT_NAME }}
-          NPM_TOKEN: ${{ secrets.SOCIALGROOVYBOT_NPM_TOKEN }}

package/.github/workflows/test.yml

@@ -1,27 +0,0 @@
-name: Test
-
-on:
-  push:
-    branches: ["*"]
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Node.js
-        uses: actions/setup-node@v2
-        with:
-          node-version: lts/*
-          cache: yarn
-
-      - name: Installing
-        run: yarn --frozen-lockfile --perfer-offline --link-duplicates
-
-      # - name: Lint
-      #   run: yarn lint
-
-      - name: Unit tests
-        run: yarn test

package/docker/Dockerfile

@@ -1,43 +0,0 @@
-FROM postgres:13-alpine
-
-ENV PG_PARTMAN_VERSION v4.7.0
-
-# Install pg_jobmon
-RUN set -ex \
-    \
-    && apk add --no-cache --virtual .fetch-deps \
-    ca-certificates \
-    openssl \
-    tar \
-    \
-    && apk add --no-cache --virtual .build-deps \
-    autoconf \
-    automake \
-    g++ \
-    clang \
-    llvm \
-    libtool \
-    libxml2-dev \
-    make \
-    perl
-# Install pg_partman
-RUN set -ex \
-    && wget -O pg_partman.tar.gz "https://github.com/pgpartman/pg_partman/archive/$PG_PARTMAN_VERSION.tar.gz" \
-    && mkdir -p /usr/src/pg_partman \
-    && tar \
-    --extract \
-    --file pg_partman.tar.gz \
-    --directory /usr/src/pg_partman \
-    --strip-components 1 \
-    && rm pg_partman.tar.gz \
-    && cd /usr/src/pg_partman \
-    && make \
-    && make install \
-    && cd / \
-    && rm -rf /usr/src/pg_partman \
-    && apk del .fetch-deps .build-deps
-
-# Copy the init script
-# The Docker Postgres initd script will run anything 
-# in the directory /docker-entrypoint-initdb.d
-COPY initdb.sh /docker-entrypoint-initdb.d/initdb.sh

package/docker/initdb.sh

@@ -1,13 +0,0 @@
-#!/bin/bash -e
-
-echo "Creating partman extension"
-psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
-    CREATE SCHEMA partman;
-    CREATE EXTENSION pg_partman SCHEMA partman;
-EOSQL
-
-echo "ADDING pg_partman_bgw TO postgresql.conf"
-echo "shared_preload_libraries = 'pg_partman_bgw'" >> $PGDATA/postgresql.conf
-echo "pg_partman_bgw.interval = 3600" >> $PGDATA/postgresql.conf
-echo "pg_partman_bgw.role = '$POSTGRES_USER'" >> $PGDATA/postgresql.conf
-echo "pg_partman_bgw.dbname = '$POSTGRES_DB'" >> $PGDATA/postgresql.conf

package/docker-compose.yml

@@ -1,18 +0,0 @@
-version: "3.0"
-services:
-  postgres:
-    build:
-      context: ./docker
-      dockerfile: ./Dockerfile
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    environment:
-      POSTGRES_PASSWORD: postgres
-      POSTGRES_USERNAME: postgres
-      TZ: "Europe/Paris"
-    ports:
-      - 5455:5432
-
-
-volumes:
-  postgres_data:

package/initial.sql

@@ -1,73 +0,0 @@
--- converting existing matomo table to partitioned witg pg_partman
--- usage : ON_ERROR_STOP=1 psql < partition.sql
-
---- pg_partman setup
-
-CREATE SCHEMA IF NOT EXISTS partman;
-CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
-
---- backup and recreate a new partionned matomo table
-
-CREATE TABLE IF NOT EXISTS matomo_tmp as (select * from matomo);
-
-ALTER TABLE IF EXISTS matomo RENAME TO matomo_backup;
-
-CREATE TABLE IF NOT EXISTS matomo
-(
-    idsite                      text,
-    idvisit                     text,
-    actions                     text,
-    country                     text,
-    region                      text,
-    city                        text,
-    operatingsystemname         text,
-    devicemodel                 text,
-    devicebrand                 text,
-    visitduration               text,
-    dayssincefirstvisit         text,
-    visitortype                 text,
-    sitename                    text,
-    userid                      text,
-    serverdateprettyfirstaction date,
-    action_id                   text,
-    action_type                 text,
-    action_eventcategory        text,
-    action_eventaction          text,
-    action_eventname            text,
-    action_eventvalue           decimal,
-    action_timespent            text,
-    action_timestamp            timestamp with time zone DEFAULT now(),
-    usercustomproperties        json,
-    usercustomdimensions        json,
-    dimension1                  text,
-    dimension2                  text,
-    dimension3                  text,
-    dimension4                  text,
-    dimension5                  text,
-    dimension6                  text,
-    dimension7                  text,
-    dimension8                  text,
-    dimension9                  text,
-    dimension10                 text,
-    action_url                  text,
-    sitesearchkeyword           text,
-    action_title                text
-) PARTITION BY RANGE (action_timestamp);
-
-ALTER TABLE IF EXISTS matomo ADD CONSTRAINT unique_action_id UNIQUE (action_id, action_timestamp);
-ALTER TABLE IF EXISTS matomo ALTER COLUMN action_eventvalue TYPE decimal USING action_eventvalue::decimal;
-CREATE INDEX IF NOT EXISTS idx_action_timestamp_matomo ON matomo (action_timestamp);
-CREATE INDEX IF NOT EXISTS idx_idvisit_matomo ON matomo(idvisit);
-CREATE INDEX IF NOT EXISTS idx_action_eventcategory_matomo ON matomo(action_eventcategory);
-CREATE INDEX IF NOT EXISTS idx_action_type_matomo ON matomo(action_type);
-CREATE INDEX IF NOT EXISTS idx_action_eventaction_matomo ON matomo(action_eventaction);
-
-SELECT partman.create_parent('public.matomo', 'action_timestamp', 'native', 'monthly');
-
--- Import des données depuis la table standard vers la table partitionnée
-CALL partman.partition_data_proc('public.matomo', p_source_table := 'public.matomo_tmp', p_order:= 'DESC');
-
-VACUUM ANALYZE public.matomo;
-
-DROP TABLE if exists matomo_tmp;
-

package/jsconfig.json

@@ -1,10 +0,0 @@
-{
-  "compilerOptions": {
-    "baseUrl": ".",
-    "resolveJsonModule": true,
-    "checkJs": true,
-    "module": "commonjs",
-    "target": "es6"
-  },
-  "include": ["src/**/*", "types/*"]
-}

package/renovate.json

@@ -1,6 +0,0 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "local>SocialGouv/renovate-config:light"
-  ]
-}