@socialgouv/matomo-postgres 2.3.14 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -213,6 +213,14 @@ Migrations run automatically on each `pnpm start` to ensure schema compatibility
213
213
  - Parallel event insertion (configurable)
214
214
  - Automatic pagination for large datasets
215
215
 
216
+ ## ⚠️ Limitations
217
+
218
+ ### Actions per visit cap
219
+
220
+ The Matomo `Live.getLastVisitsDetails` API limits the number of actions returned per visit to **99**.
221
+ If a user performed more than **99** actions during a visit, the extra actions will be missing from the database (see [issue #92](https://github.com/SocialGouv/matomo-postgres/issues/92)).
222
+ This limitation comes from Matomo’s API itself; this library does not implement any per-action workaround.
223
+
216
224
  ## 🐛 Troubleshooting
217
225
 
218
226
  ### Common Issues
@@ -17,6 +17,16 @@
17
17
  "region": "Buenos Aires",
18
18
  "city": "Buenos Aires",
19
19
  "resolution": "1920x1080",
20
+ "experiments": [
21
+ {
22
+ "idexperiment": "3",
23
+ "name": "search_ab_test",
24
+ "variation": {
25
+ "idvariation": 5,
26
+ "name": "search_v2"
27
+ }
28
+ }
29
+ ],
20
30
  "dimension1": "guest",
21
31
  "dimension3": "page",
22
32
  "dimension6": "shop",
@@ -52,7 +52,8 @@ const MATOMO_INSERT_COLUMNS = [
52
52
  'visitorid',
53
53
  'referrertype',
54
54
  'referrername',
55
- 'resolution'
55
+ 'resolution',
56
+ 'experiments'
56
57
  ];
57
58
  const MATOMO_INSERT_COLUMN_SQL = sql.join(MATOMO_INSERT_COLUMNS.map((column) => sql.id(column)), sql `,\n`);
58
59
  /**
@@ -63,7 +64,7 @@ const MATOMO_INSERT_COLUMN_SQL = sql.join(MATOMO_INSERT_COLUMNS.map((column) =>
63
64
  * @return {Promise<void>}
64
65
  */
65
66
  export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
66
- var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
67
+ var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15;
67
68
  // Build a sanitized, typed data object to reduce drift and ensure defaults in one place
68
69
  const eventData = {
69
70
  action_id: (_a = event.action_id) !== null && _a !== void 0 ? _a : '',
@@ -113,7 +114,8 @@ export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function
113
114
  visitorid: (_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null,
114
115
  referrertype: (_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null,
115
116
  referrername: (_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null,
116
- resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null
117
+ resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null,
118
+ experiments: (_15 = event.experiments) !== null && _15 !== void 0 ? _15 : null
117
119
  };
118
120
  // Minimal runtime validation for required fields
119
121
  if (!eventData.action_id || eventData.action_id.trim().length === 0) {
@@ -197,7 +199,7 @@ const actionProps = {
197
199
  };
198
200
  export const getEventsFromMatomoVisit = (matomoVisit) => {
199
201
  return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
200
- var _a;
202
+ var _a, _b;
201
203
  const usercustomproperties = {};
202
204
  for (let k = 1; k < 10; k++) {
203
205
  const property = (_a = actionDetail.customVariables) === null || _a === void 0 ? void 0 : _a[k];
@@ -219,7 +221,7 @@ export const getEventsFromMatomoVisit = (matomoVisit) => {
219
221
  //@ts-expect-error implicit any type
220
222
  usercustomdimensions[dimension] = value;
221
223
  }
222
- const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
224
+ const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { experiments: (_b = matomoVisit.experiments) !== null && _b !== void 0 ? _b : null, serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
223
225
  action_id: `${matomoVisit.idVisit}_${actionIndex}`
224
226
  })), {
225
227
  // custom variables
@@ -0,0 +1,343 @@
1
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
+ return new (P || (P = Promise))(function (resolve, reject) {
4
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
8
+ });
9
+ };
10
+ import { sql } from 'kysely';
11
+ const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
12
+ const PARTITIONED_MATOMO_TABLE_NAME = process.env.PARTITIONED_MATOMO_TABLE_NAME || 'matomo_partitioned';
13
+ export function up(db) {
14
+ return __awaiter(this, void 0, void 0, function* () {
15
+ // Add column for AB Testing
16
+ yield db.schema
17
+ .alterTable(MATOMO_TABLE_NAME)
18
+ .addColumn('experiments', 'jsonb')
19
+ .execute();
20
+ yield db.schema
21
+ .alterTable(PARTITIONED_MATOMO_TABLE_NAME)
22
+ .addColumn('experiments', 'jsonb')
23
+ .execute();
24
+ // Drop the previous version of the stored procedure so the signature can grow
25
+ // (CREATE OR REPLACE refuses to change parameter lists in PL/pgSQL).
26
+ yield sql `DROP FUNCTION IF EXISTS insert_into_matomo_partitioned(
27
+ text, timestamptz, text, text, text, text, text, text, text, text, text,
28
+ text, text, text, text, text, date, text, text, text, text, numeric, text,
29
+ json, json, text, text, text, text, text, text, text, text, text, text,
30
+ text, text, text, text, text, text, text
31
+ )`.execute(db);
32
+ yield sql `
33
+ CREATE OR REPLACE FUNCTION insert_into_matomo_partitioned(
34
+ p_action_id text,
35
+ p_action_timestamp timestamptz,
36
+ p_idsite text DEFAULT '',
37
+ p_idvisit text DEFAULT '',
38
+ p_actions text DEFAULT NULL,
39
+ p_country text DEFAULT NULL,
40
+ p_region text DEFAULT NULL,
41
+ p_city text DEFAULT NULL,
42
+ p_operatingsystemname text DEFAULT NULL,
43
+ p_devicemodel text DEFAULT NULL,
44
+ p_devicebrand text DEFAULT NULL,
45
+ p_visitduration text DEFAULT NULL,
46
+ p_dayssincefirstvisit text DEFAULT NULL,
47
+ p_visitortype text DEFAULT NULL,
48
+ p_sitename text DEFAULT NULL,
49
+ p_userid text DEFAULT NULL,
50
+ p_serverdateprettyfirstaction date DEFAULT NULL,
51
+ p_action_type text DEFAULT '',
52
+ p_action_eventcategory text DEFAULT '',
53
+ p_action_eventaction text DEFAULT '',
54
+ p_action_eventname text DEFAULT '',
55
+ p_action_eventvalue numeric DEFAULT 0,
56
+ p_action_timespent text DEFAULT '0',
57
+ p_usercustomproperties json DEFAULT NULL,
58
+ p_usercustomdimensions json DEFAULT NULL,
59
+ p_dimension1 text DEFAULT NULL,
60
+ p_dimension2 text DEFAULT NULL,
61
+ p_dimension3 text DEFAULT NULL,
62
+ p_dimension4 text DEFAULT NULL,
63
+ p_dimension5 text DEFAULT NULL,
64
+ p_dimension6 text DEFAULT NULL,
65
+ p_dimension7 text DEFAULT NULL,
66
+ p_dimension8 text DEFAULT NULL,
67
+ p_dimension9 text DEFAULT NULL,
68
+ p_dimension10 text DEFAULT NULL,
69
+ p_action_url text DEFAULT NULL,
70
+ p_sitesearchkeyword text DEFAULT NULL,
71
+ p_action_title text DEFAULT NULL,
72
+ p_visitorid text DEFAULT NULL,
73
+ p_referrertype text DEFAULT NULL,
74
+ p_referrername text DEFAULT NULL,
75
+ p_resolution text DEFAULT NULL,
76
+ p_experiments jsonb DEFAULT NULL
77
+ )
78
+ RETURNS void
79
+ LANGUAGE plpgsql
80
+ SECURITY INVOKER
81
+ AS $$
82
+ BEGIN
83
+ PERFORM create_weekly_partition_if_not_exists('${sql.raw(PARTITIONED_MATOMO_TABLE_NAME)}', p_action_timestamp);
84
+
85
+ INSERT INTO ${sql.id(PARTITIONED_MATOMO_TABLE_NAME)} (
86
+ action_id,
87
+ action_timestamp,
88
+ idsite,
89
+ idvisit,
90
+ actions,
91
+ country,
92
+ region,
93
+ city,
94
+ operatingsystemname,
95
+ devicemodel,
96
+ devicebrand,
97
+ visitduration,
98
+ dayssincefirstvisit,
99
+ visitortype,
100
+ sitename,
101
+ userid,
102
+ serverdateprettyfirstaction,
103
+ action_type,
104
+ action_eventcategory,
105
+ action_eventaction,
106
+ action_eventname,
107
+ action_eventvalue,
108
+ action_timespent,
109
+ usercustomproperties,
110
+ usercustomdimensions,
111
+ dimension1,
112
+ dimension2,
113
+ dimension3,
114
+ dimension4,
115
+ dimension5,
116
+ dimension6,
117
+ dimension7,
118
+ dimension8,
119
+ dimension9,
120
+ dimension10,
121
+ action_url,
122
+ sitesearchkeyword,
123
+ action_title,
124
+ visitorid,
125
+ referrertype,
126
+ referrername,
127
+ resolution,
128
+ experiments
129
+ ) VALUES (
130
+ p_action_id,
131
+ p_action_timestamp,
132
+ p_idsite,
133
+ p_idvisit,
134
+ p_actions,
135
+ p_country,
136
+ p_region,
137
+ p_city,
138
+ p_operatingsystemname,
139
+ p_devicemodel,
140
+ p_devicebrand,
141
+ p_visitduration,
142
+ p_dayssincefirstvisit,
143
+ p_visitortype,
144
+ p_sitename,
145
+ p_userid,
146
+ p_serverdateprettyfirstaction,
147
+ p_action_type,
148
+ p_action_eventcategory,
149
+ p_action_eventaction,
150
+ p_action_eventname,
151
+ p_action_eventvalue,
152
+ p_action_timespent,
153
+ p_usercustomproperties,
154
+ p_usercustomdimensions,
155
+ p_dimension1,
156
+ p_dimension2,
157
+ p_dimension3,
158
+ p_dimension4,
159
+ p_dimension5,
160
+ p_dimension6,
161
+ p_dimension7,
162
+ p_dimension8,
163
+ p_dimension9,
164
+ p_dimension10,
165
+ p_action_url,
166
+ p_sitesearchkeyword,
167
+ p_action_title,
168
+ p_visitorid,
169
+ p_referrertype,
170
+ p_referrername,
171
+ p_resolution,
172
+ p_experiments
173
+ )
174
+ ON CONFLICT (action_id, action_timestamp) DO NOTHING;
175
+ END;
176
+ $$;
177
+ `.execute(db);
178
+ });
179
+ }
180
+ export function down(db) {
181
+ return __awaiter(this, void 0, void 0, function* () {
182
+ // Drop the new version of the stored procedure
183
+ yield sql `DROP FUNCTION IF EXISTS insert_into_matomo_partitioned(
184
+ text, timestamptz, text, text, text, text, text, text, text, text, text,
185
+ text, text, text, text, text, date, text, text, text, text, numeric, text,
186
+ json, json, text, text, text, text, text, text, text, text, text, text,
187
+ text, text, text, text, text, text, text, jsonb
188
+ )`.execute(db);
189
+ // Restore the previous version (without experiments) so the downgrade is usable
190
+ yield sql `
191
+ CREATE OR REPLACE FUNCTION insert_into_matomo_partitioned(
192
+ p_action_id text,
193
+ p_action_timestamp timestamptz,
194
+ p_idsite text DEFAULT '',
195
+ p_idvisit text DEFAULT '',
196
+ p_actions text DEFAULT NULL,
197
+ p_country text DEFAULT NULL,
198
+ p_region text DEFAULT NULL,
199
+ p_city text DEFAULT NULL,
200
+ p_operatingsystemname text DEFAULT NULL,
201
+ p_devicemodel text DEFAULT NULL,
202
+ p_devicebrand text DEFAULT NULL,
203
+ p_visitduration text DEFAULT NULL,
204
+ p_dayssincefirstvisit text DEFAULT NULL,
205
+ p_visitortype text DEFAULT NULL,
206
+ p_sitename text DEFAULT NULL,
207
+ p_userid text DEFAULT NULL,
208
+ p_serverdateprettyfirstaction date DEFAULT NULL,
209
+ p_action_type text DEFAULT '',
210
+ p_action_eventcategory text DEFAULT '',
211
+ p_action_eventaction text DEFAULT '',
212
+ p_action_eventname text DEFAULT '',
213
+ p_action_eventvalue numeric DEFAULT 0,
214
+ p_action_timespent text DEFAULT '0',
215
+ p_usercustomproperties json DEFAULT NULL,
216
+ p_usercustomdimensions json DEFAULT NULL,
217
+ p_dimension1 text DEFAULT NULL,
218
+ p_dimension2 text DEFAULT NULL,
219
+ p_dimension3 text DEFAULT NULL,
220
+ p_dimension4 text DEFAULT NULL,
221
+ p_dimension5 text DEFAULT NULL,
222
+ p_dimension6 text DEFAULT NULL,
223
+ p_dimension7 text DEFAULT NULL,
224
+ p_dimension8 text DEFAULT NULL,
225
+ p_dimension9 text DEFAULT NULL,
226
+ p_dimension10 text DEFAULT NULL,
227
+ p_action_url text DEFAULT NULL,
228
+ p_sitesearchkeyword text DEFAULT NULL,
229
+ p_action_title text DEFAULT NULL,
230
+ p_visitorid text DEFAULT NULL,
231
+ p_referrertype text DEFAULT NULL,
232
+ p_referrername text DEFAULT NULL,
233
+ p_resolution text DEFAULT NULL
234
+ )
235
+ RETURNS void
236
+ LANGUAGE plpgsql
237
+ SECURITY INVOKER
238
+ AS $$
239
+ BEGIN
240
+ PERFORM create_weekly_partition_if_not_exists('${sql.raw(PARTITIONED_MATOMO_TABLE_NAME)}', p_action_timestamp);
241
+
242
+ INSERT INTO ${sql.id(PARTITIONED_MATOMO_TABLE_NAME)} (
243
+ action_id,
244
+ action_timestamp,
245
+ idsite,
246
+ idvisit,
247
+ actions,
248
+ country,
249
+ region,
250
+ city,
251
+ operatingsystemname,
252
+ devicemodel,
253
+ devicebrand,
254
+ visitduration,
255
+ dayssincefirstvisit,
256
+ visitortype,
257
+ sitename,
258
+ userid,
259
+ serverdateprettyfirstaction,
260
+ action_type,
261
+ action_eventcategory,
262
+ action_eventaction,
263
+ action_eventname,
264
+ action_eventvalue,
265
+ action_timespent,
266
+ usercustomproperties,
267
+ usercustomdimensions,
268
+ dimension1,
269
+ dimension2,
270
+ dimension3,
271
+ dimension4,
272
+ dimension5,
273
+ dimension6,
274
+ dimension7,
275
+ dimension8,
276
+ dimension9,
277
+ dimension10,
278
+ action_url,
279
+ sitesearchkeyword,
280
+ action_title,
281
+ visitorid,
282
+ referrertype,
283
+ referrername,
284
+ resolution
285
+ ) VALUES (
286
+ p_action_id,
287
+ p_action_timestamp,
288
+ p_idsite,
289
+ p_idvisit,
290
+ p_actions,
291
+ p_country,
292
+ p_region,
293
+ p_city,
294
+ p_operatingsystemname,
295
+ p_devicemodel,
296
+ p_devicebrand,
297
+ p_visitduration,
298
+ p_dayssincefirstvisit,
299
+ p_visitortype,
300
+ p_sitename,
301
+ p_userid,
302
+ p_serverdateprettyfirstaction,
303
+ p_action_type,
304
+ p_action_eventcategory,
305
+ p_action_eventaction,
306
+ p_action_eventname,
307
+ p_action_eventvalue,
308
+ p_action_timespent,
309
+ p_usercustomproperties,
310
+ p_usercustomdimensions,
311
+ p_dimension1,
312
+ p_dimension2,
313
+ p_dimension3,
314
+ p_dimension4,
315
+ p_dimension5,
316
+ p_dimension6,
317
+ p_dimension7,
318
+ p_dimension8,
319
+ p_dimension9,
320
+ p_dimension10,
321
+ p_action_url,
322
+ p_sitesearchkeyword,
323
+ p_action_title,
324
+ p_visitorid,
325
+ p_referrertype,
326
+ p_referrername,
327
+ p_resolution
328
+ )
329
+ ON CONFLICT (action_id, action_timestamp) DO NOTHING;
330
+ END;
331
+ $$;
332
+ `.execute(db);
333
+ // Drop column for AB Testing
334
+ yield db.schema
335
+ .alterTable(MATOMO_TABLE_NAME)
336
+ .dropColumn('experiments')
337
+ .execute();
338
+ yield db.schema
339
+ .alterTable(PARTITIONED_MATOMO_TABLE_NAME)
340
+ .dropColumn('experiments')
341
+ .execute();
342
+ });
343
+ }
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@socialgouv/matomo-postgres",
3
3
  "description": "Extract visitor events from Matomo API and push to Postgres",
4
- "version": "2.3.14",
4
+ "version": "2.4.0",
5
5
  "packageManager": "pnpm@10.28.1",
6
6
  "types": "types/index.d.ts",
7
7
  "license": "Apache-2.0",