@socialgouv/matomo-postgres 1.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ name: Release
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ branches: [master, alpha, beta, next]
7
+
8
+ jobs:
9
+ release:
10
+ name: Release
11
+ runs-on: ubuntu-latest
12
+ steps:
13
+ - name: Checkout repository
14
+ uses: actions/checkout@v2
15
+ with:
16
+ fetch-depth: 0
17
+ persist-credentials: false
18
+
19
+ - name: Semantic Release
20
+ uses: cycjimmy/semantic-release-action@v2
21
+ with:
22
+ semantic_version: 17
23
+ extra_plugins: |
24
+ @semantic-release/changelog@5.0.1
25
+ @semantic-release/git@9.0.0
26
+
27
+ env:
28
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
29
+ GIT_AUTHOR_EMAIL: ${{ secrets.SOCIALGROOVYBOT_EMAIL }}
30
+ GIT_AUTHOR_NAME: ${{ secrets.SOCIALGROOVYBOT_NAME }}
31
+ GIT_COMMITTER_EMAIL: ${{ secrets.SOCIALGROOVYBOT_EMAIL }}
32
+ GIT_COMMITTER_NAME: ${{ secrets.SOCIALGROOVYBOT_NAME }}
33
+ NPM_TOKEN: ${{ secrets.SOCIALGROOVYBOT_NPM_TOKEN }}
@@ -0,0 +1,27 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches: ["*"]
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+
11
+ steps:
12
+ - uses: actions/checkout@v2
13
+
14
+ - name: Set up Node.js
15
+ uses: actions/setup-node@v2
16
+ with:
17
+ node-version: lts/*
18
+ cache: yarn
19
+
20
+ - name: Installing
21
+ run: yarn --frozen-lockfile --perfer-offline --link-duplicates
22
+
23
+ # - name: Lint
24
+ # run: yarn lint
25
+
26
+ - name: Unit tests
27
+ run: yarn test
package/README.md ADDED
@@ -0,0 +1,40 @@
1
+ # @socialgouv/matomo-postgres
2
+
3
+ Extract matomo data and push to Postgres
4
+
5
+ Use Matomo [`Live.getLastVisitsDetails`](https://developer.matomo.org/api-reference/reporting-api) API to extract visits informations.
6
+
7
+ ## Usage
8
+
9
+ ```sh
10
+ npx @socialgouv/matomo-postgres
11
+ ```
12
+
13
+ ### Environment variables Deployment
14
+
15
+ | name | value |
16
+ | ----------------- | ---------------------------------------------- |
17
+ | MATOMO_KEY\* | matomo api token |
18
+ | MATOMO_SITE\* | matomo site id |
19
+ | MATOMO_URL\* | matomo url |
20
+ | PGDATABASE\* | Postgres connection string |
21
+ | DESTINATION_TABLE | `matomo` |
22
+ | STARTDATE | default to today() |
23
+ | RESULTPERPAGE | matomo pagination : `100` |
24
+ | OFFSET | default days to check in the past; default = 3 |
25
+
26
+ ## Dev
27
+
28
+ ```sh
29
+ docker-compose up
30
+ export MATOMO_URL=
31
+ export MATOMO_SITE=
32
+ export MATOMO_KEY=
33
+ export DESTINATION_TABLE=
34
+ export STARTDATE=
35
+ export OFFSET=
36
+ export PGDATABASE=postgres://postgres:postgres@127.0.0.1:5455/postgres
37
+ yarn start
38
+ ```
39
+
40
+ Use `yarn test -u` to update the snapshots
package/bin/index.js ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env node
2
+
3
+ const run = require("../src/index");
4
+
5
+ if (require.main === module) {
6
+ const date =
7
+ (process.argv[process.argv.length - 1].match(/^\d\d\d\d-\d\d-\d\d$/) && process.argv[process.argv.length - 1]) ||
8
+ "";
9
+ console.log(`\nRunning @socialgouv/matomo-metabase ${date}\n`);
10
+ run(date);
11
+ }
@@ -0,0 +1,10 @@
1
+ version: "3.0"
2
+ services:
3
+ postgres:
4
+ image: postgres:11
5
+ environment:
6
+ POSTGRES_PASSWORD: postgres
7
+ POSTGRES_USERNAME: postgres
8
+ TZ: "Europe/Paris"
9
+ ports:
10
+ - 5455:5432
package/jsconfig.json ADDED
@@ -0,0 +1,10 @@
1
+ {
2
+ "compilerOptions": {
3
+ "baseUrl": ".",
4
+ "resolveJsonModule": true,
5
+ "checkJs": true,
6
+ "module": "commonjs",
7
+ "target": "es6"
8
+ },
9
+ "include": ["src/**/*", "types/*"]
10
+ }
package/package.json ADDED
@@ -0,0 +1,36 @@
1
+ {
2
+ "name": "@socialgouv/matomo-postgres",
3
+ "description": "Extract visitor events from Matomo API and push to Postgres",
4
+ "version": "1.1.4",
5
+ "types": "types/index.d.ts",
6
+ "license": "Apache-2.0",
7
+ "main": "src/index.js",
8
+ "preferGlobal": true,
9
+ "publishConfig": {
10
+ "access": "public"
11
+ },
12
+ "bin": {
13
+ "matomo-metabase": "./bin/index.js"
14
+ },
15
+ "scripts": {
16
+ "start": "node ./src/index.js",
17
+ "test": "jest --verbose"
18
+ },
19
+ "prettier": {
20
+ "printWidth": 120
21
+ },
22
+ "dependencies": {
23
+ "date-fns": "^2.23.0",
24
+ "debug": "^4.3.3",
25
+ "dotenv": "^10.0.0",
26
+ "p-all": "^3.0.0",
27
+ "pg": "^8.7.1",
28
+ "piwik-client": "^0.2.2"
29
+ },
30
+ "devDependencies": {
31
+ "@types/jest": "^27.0.3",
32
+ "@types/pg": "^8.6.1",
33
+ "jest": "^27.1.1",
34
+ "typescript": "^4.4.4"
35
+ }
36
+ }
package/renovate.json ADDED
@@ -0,0 +1,6 @@
1
+ {
2
+ "$schema": "https://docs.renovatebot.com/renovate-schema.json",
3
+ "extends": [
4
+ "local>SocialGouv/renovate-config:light"
5
+ ]
6
+ }
@@ -0,0 +1,189 @@
1
+ // Jest Snapshot v1, https://goo.gl/fbAQLP
2
+
3
+ exports[`importDate: should import all events for a given date 1`] = `
4
+ Array [
5
+ Object {
6
+ "date": "2021-08-02",
7
+ "filter_limit": 500,
8
+ "filter_offset": 0,
9
+ "filter_sort_order": "asc",
10
+ "idSite": 0,
11
+ "method": "Live.getLastVisitsDetails",
12
+ "period": "day",
13
+ },
14
+ ]
15
+ `;
16
+
17
+ exports[`importDate: should import all events for a given date 2`] = `
18
+ Array [
19
+ Array [
20
+ "SELECT COUNT(distinct idvisit) FROM matomo WHERE action_timestamp::date='2021-08-02';",
21
+ ],
22
+ Array [
23
+ "insert into matomo
24
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
25
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
26
+ ON CONFLICT DO NOTHING",
27
+ Array [
28
+ "42",
29
+ 123,
30
+ "2",
31
+ "Argentine",
32
+ "Buenos Aires",
33
+ "Buenos Aires",
34
+ "Mac",
35
+ "Générique Bureau",
36
+ "Inconnu",
37
+ "300",
38
+ "23",
39
+ "returningCustomer",
40
+ "tests",
41
+ "24",
42
+ "2021-08-20T21:55:12.000Z",
43
+ "123_0",
44
+ "event",
45
+ "Ecommerce",
46
+ "Cart change",
47
+ "added - Basic Wetsuit",
48
+ 1,
49
+ 48,
50
+ "2021-08-20T21:35:18.000Z",
51
+ "https://dive-shop.net/products/basic-wetsuit/",
52
+ Object {
53
+ "page-author": "Julien",
54
+ "post-age": "-430 days",
55
+ },
56
+ Object {
57
+ "dimension1": "guest",
58
+ "dimension2": "julien",
59
+ "dimension3": "page",
60
+ "dimension4": "indonesia",
61
+ "dimension5": "diving",
62
+ },
63
+ ],
64
+ ],
65
+ Array [
66
+ "insert into matomo
67
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
68
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
69
+ ON CONFLICT DO NOTHING",
70
+ Array [
71
+ "42",
72
+ 123,
73
+ "2",
74
+ "Argentine",
75
+ "Buenos Aires",
76
+ "Buenos Aires",
77
+ "Mac",
78
+ "Générique Bureau",
79
+ "Inconnu",
80
+ "300",
81
+ "23",
82
+ "returningCustomer",
83
+ "tests",
84
+ "24",
85
+ "2021-08-20T21:55:12.000Z",
86
+ "123_1",
87
+ "action",
88
+ undefined,
89
+ undefined,
90
+ undefined,
91
+ undefined,
92
+ "2",
93
+ "2021-08-20T21:30:25.000Z",
94
+ "https://dive-shop.net/products/diving-boots/",
95
+ Object {},
96
+ Object {
97
+ "dimension1": "guest",
98
+ "dimension2": "julien",
99
+ "dimension3": "page",
100
+ "dimension4": "indonesia",
101
+ "dimension5": "diving",
102
+ },
103
+ ],
104
+ ],
105
+ Array [
106
+ "insert into matomo
107
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
108
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
109
+ ON CONFLICT DO NOTHING",
110
+ Array [
111
+ "42",
112
+ 124,
113
+ "2",
114
+ "Argentine",
115
+ "Buenos Aires",
116
+ "Buenos Aires",
117
+ "Mac",
118
+ "Générique Bureau",
119
+ "Inconnu",
120
+ "300",
121
+ "23",
122
+ "returningCustomer",
123
+ "tests",
124
+ "24",
125
+ "2021-08-20T21:55:12.000Z",
126
+ "124_0",
127
+ "event",
128
+ "Ecommerce",
129
+ "Cart change",
130
+ "added - Basic Wetsuit",
131
+ 1,
132
+ 48,
133
+ "2021-08-20T21:35:18.000Z",
134
+ "https://dive-shop.net/products/basic-wetsuit/",
135
+ Object {
136
+ "page-author": "Julien",
137
+ "post-age": "-430 days",
138
+ },
139
+ Object {
140
+ "dimension1": "guest",
141
+ "dimension2": "julien",
142
+ "dimension3": "page",
143
+ "dimension4": "indonesia",
144
+ "dimension5": "diving",
145
+ },
146
+ ],
147
+ ],
148
+ Array [
149
+ "insert into matomo
150
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
151
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
152
+ ON CONFLICT DO NOTHING",
153
+ Array [
154
+ "42",
155
+ 124,
156
+ "2",
157
+ "Argentine",
158
+ "Buenos Aires",
159
+ "Buenos Aires",
160
+ "Mac",
161
+ "Générique Bureau",
162
+ "Inconnu",
163
+ "300",
164
+ "23",
165
+ "returningCustomer",
166
+ "tests",
167
+ "24",
168
+ "2021-08-20T21:55:12.000Z",
169
+ "124_1",
170
+ "action",
171
+ undefined,
172
+ undefined,
173
+ undefined,
174
+ undefined,
175
+ "2",
176
+ "2021-08-20T21:30:25.000Z",
177
+ "https://dive-shop.net/products/diving-boots/",
178
+ Object {},
179
+ Object {
180
+ "dimension1": "guest",
181
+ "dimension2": "julien",
182
+ "dimension3": "page",
183
+ "dimension4": "indonesia",
184
+ "dimension5": "diving",
185
+ },
186
+ ],
187
+ ],
188
+ ]
189
+ `;
@@ -0,0 +1,165 @@
1
+ // Jest Snapshot v1, https://goo.gl/fbAQLP
2
+
3
+ exports[`importEvent: should extract events from matomo visit actionsDetails and create insert queries 1`] = `
4
+ Array [
5
+ Object {
6
+ "action_eventaction": "Cart change",
7
+ "action_eventcategory": "Ecommerce",
8
+ "action_eventname": "added - Basic Wetsuit",
9
+ "action_eventvalue": 1,
10
+ "action_id": "124_0",
11
+ "action_timespent": 48,
12
+ "action_timestamp": "2021-08-20T21:35:18.000Z",
13
+ "action_type": "event",
14
+ "action_url": "https://dive-shop.net/products/basic-wetsuit/",
15
+ "actions": "2",
16
+ "city": "Buenos Aires",
17
+ "country": "Argentine",
18
+ "dayssincefirstvisit": "23",
19
+ "devicebrand": "Inconnu",
20
+ "devicemodel": "Générique Bureau",
21
+ "idsite": "42",
22
+ "idvisit": "124",
23
+ "operatingsystemname": "Mac",
24
+ "region": "Buenos Aires",
25
+ "serverdateprettyfirstaction": "2021-08-20T21:55:12.000Z",
26
+ "sitename": "tests",
27
+ "usercustomdimensions": Object {
28
+ "dimension1": "guest",
29
+ "dimension2": "julien",
30
+ "dimension3": "page",
31
+ "dimension4": "indonesia",
32
+ "dimension5": "diving",
33
+ },
34
+ "usercustomproperties": Object {
35
+ "page-author": "Julien",
36
+ "post-age": "-430 days",
37
+ },
38
+ "userid": "24",
39
+ "visitduration": "300",
40
+ "visitortype": "returningCustomer",
41
+ },
42
+ Object {
43
+ "action_eventaction": undefined,
44
+ "action_eventcategory": undefined,
45
+ "action_eventname": undefined,
46
+ "action_eventvalue": undefined,
47
+ "action_id": "124_1",
48
+ "action_timespent": "2",
49
+ "action_timestamp": "2021-08-20T21:30:25.000Z",
50
+ "action_type": "action",
51
+ "action_url": "https://dive-shop.net/products/diving-boots/",
52
+ "actions": "2",
53
+ "city": "Buenos Aires",
54
+ "country": "Argentine",
55
+ "dayssincefirstvisit": "23",
56
+ "devicebrand": "Inconnu",
57
+ "devicemodel": "Générique Bureau",
58
+ "idsite": "42",
59
+ "idvisit": "124",
60
+ "operatingsystemname": "Mac",
61
+ "region": "Buenos Aires",
62
+ "serverdateprettyfirstaction": "2021-08-20T21:55:12.000Z",
63
+ "sitename": "tests",
64
+ "usercustomdimensions": Object {
65
+ "dimension1": "guest",
66
+ "dimension2": "julien",
67
+ "dimension3": "page",
68
+ "dimension4": "indonesia",
69
+ "dimension5": "diving",
70
+ },
71
+ "usercustomproperties": Object {},
72
+ "userid": "24",
73
+ "visitduration": "300",
74
+ "visitortype": "returningCustomer",
75
+ },
76
+ ]
77
+ `;
78
+
79
+ exports[`importEvent: should extract events from matomo visit actionsDetails and create insert queries 2`] = `
80
+ Array [
81
+ Array [
82
+ "insert into matomo
83
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
84
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
85
+ ON CONFLICT DO NOTHING",
86
+ Array [
87
+ "42",
88
+ "124",
89
+ "2",
90
+ "Argentine",
91
+ "Buenos Aires",
92
+ "Buenos Aires",
93
+ "Mac",
94
+ "Générique Bureau",
95
+ "Inconnu",
96
+ "300",
97
+ "23",
98
+ "returningCustomer",
99
+ "tests",
100
+ "24",
101
+ "2021-08-20T21:55:12.000Z",
102
+ "124_0",
103
+ "event",
104
+ "Ecommerce",
105
+ "Cart change",
106
+ "added - Basic Wetsuit",
107
+ 1,
108
+ 48,
109
+ "2021-08-20T21:35:18.000Z",
110
+ "https://dive-shop.net/products/basic-wetsuit/",
111
+ Object {
112
+ "page-author": "Julien",
113
+ "post-age": "-430 days",
114
+ },
115
+ Object {
116
+ "dimension1": "guest",
117
+ "dimension2": "julien",
118
+ "dimension3": "page",
119
+ "dimension4": "indonesia",
120
+ "dimension5": "diving",
121
+ },
122
+ ],
123
+ ],
124
+ Array [
125
+ "insert into matomo
126
+ (idsite, idvisit, actions, country, region, city, operatingsystemname, devicemodel, devicebrand, visitduration, dayssincefirstvisit, visitortype, sitename, userid, serverdateprettyfirstaction, action_id, action_type, action_eventcategory, action_eventaction, action_eventname, action_eventvalue, action_timespent, action_timestamp, action_url, usercustomproperties, usercustomdimensions)
127
+ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26)
128
+ ON CONFLICT DO NOTHING",
129
+ Array [
130
+ "42",
131
+ "124",
132
+ "2",
133
+ "Argentine",
134
+ "Buenos Aires",
135
+ "Buenos Aires",
136
+ "Mac",
137
+ "Générique Bureau",
138
+ "Inconnu",
139
+ "300",
140
+ "23",
141
+ "returningCustomer",
142
+ "tests",
143
+ "24",
144
+ "2021-08-20T21:55:12.000Z",
145
+ "124_1",
146
+ "action",
147
+ undefined,
148
+ undefined,
149
+ undefined,
150
+ undefined,
151
+ "2",
152
+ "2021-08-20T21:30:25.000Z",
153
+ "https://dive-shop.net/products/diving-boots/",
154
+ Object {},
155
+ Object {
156
+ "dimension1": "guest",
157
+ "dimension2": "julien",
158
+ "dimension3": "page",
159
+ "dimension4": "indonesia",
160
+ "dimension5": "diving",
161
+ },
162
+ ],
163
+ ],
164
+ ]
165
+ `;
@@ -0,0 +1,36 @@
1
+ // Jest Snapshot v1, https://goo.gl/fbAQLP
2
+
3
+ exports[`run: should create table 1`] = `
4
+ Array [
5
+ "CREATE TABLE IF NOT EXISTS matomo
6
+ (
7
+ idsite text,
8
+ idvisit text,
9
+ actions text,
10
+ country text,
11
+ region text,
12
+ city text,
13
+ operatingsystemname text,
14
+ devicemodel text,
15
+ devicebrand text,
16
+ visitduration text,
17
+ dayssincefirstvisit text,
18
+ visitortype text,
19
+ sitename text,
20
+ userid text,
21
+ serverdateprettyfirstaction date,
22
+ action_id text UNIQUE,
23
+ action_type text,
24
+ action_eventcategory text,
25
+ action_eventaction text,
26
+ action_eventname text,
27
+ action_eventvalue text,
28
+ action_timespent text,
29
+ action_timestamp timestamp with time zone,
30
+ usercustomproperties json,
31
+ usercustomdimensions json,
32
+ action_url text
33
+ )",
34
+ Array [],
35
+ ]
36
+ `;