@socialgouv/matomo-postgres 2.3.13 → 2.4.0

package/README.md

@@ -213,6 +213,14 @@ Migrations run automatically on each `pnpm start` to ensure schema compatibility
    - Parallel event insertion (configurable)
    - Automatic pagination for large datasets
 
+## ⚠️ Limitations
+
+### Actions per visit cap
+
+The Matomo `Live.getLastVisitsDetails` API limits the number of actions returned per visit to **99**.
+If a user performed more than **99** actions during a visit, the extra actions will be missing from the database (see [issue #92](https://github.com/SocialGouv/matomo-postgres/issues/92)).
+This limitation comes from Matomo’s API itself; this library does not implement any per-action workaround.
+
 ## 🐛 Troubleshooting
 
 ### Common Issues

package/dist/__tests__/run.test.js

@@ -84,10 +84,6 @@ test('run: should run SQL queries', () => __awaiter(void 0, void 0, void 0, func
     jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
     const result = yield run();
     expect(queries).toMatchSnapshot();
-    // Updated expectation based on actual behavior with INITIAL_OFFSET=3 (5 days total: 3 days before + today + 1 day after)
-    // 5 days * (6 events per day + 1 count query per day)
-    // Note: We also capture the initial "findLastEventInMatomo" query.
-    expect(queries.length).toEqual(1 + 5 * (6 + 1));
     expect(result).toMatchObject({
         daysProcessed: 5,
         eventsImportedTotal: 5 * 6

package/dist/__tests__/visit.json

@@ -17,6 +17,16 @@
     "region": "Buenos Aires",
     "city": "Buenos Aires",
     "resolution": "1920x1080",
+    "experiments": [
+        {
+            "idexperiment": "3",
+            "name": "search_ab_test",
+            "variation": {
+                "idvariation": 5,
+                "name": "search_v2"
+            }
+        }
+    ],
     "dimension1": "guest",
     "dimension3": "page",
     "dimension6": "shop",

package/dist/importEvent.js

@@ -8,7 +8,54 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { sql } from 'kysely';
+import { DESTINATION_TABLE, PARTITIONED_MATOMO_TABLE_NAME } from './config.js';
 import { db, pool } from './db.js';
+const MATOMO_INSERT_COLUMNS = [
+    'action_id',
+    'action_timestamp',
+    'idsite',
+    'idvisit',
+    'actions',
+    'country',
+    'region',
+    'city',
+    'operatingsystemname',
+    'devicemodel',
+    'devicebrand',
+    'visitduration',
+    'dayssincefirstvisit',
+    'visitortype',
+    'sitename',
+    'userid',
+    'serverdateprettyfirstaction',
+    'action_type',
+    'action_eventcategory',
+    'action_eventaction',
+    'action_eventname',
+    'action_eventvalue',
+    'action_timespent',
+    'usercustomproperties',
+    'usercustomdimensions',
+    'dimension1',
+    'dimension2',
+    'dimension3',
+    'dimension4',
+    'dimension5',
+    'dimension6',
+    'dimension7',
+    'dimension8',
+    'dimension9',
+    'dimension10',
+    'action_url',
+    'sitesearchkeyword',
+    'action_title',
+    'visitorid',
+    'referrertype',
+    'referrername',
+    'resolution',
+    'experiments'
+];
+const MATOMO_INSERT_COLUMN_SQL = sql.join(MATOMO_INSERT_COLUMNS.map((column) => sql.id(column)), sql `,\n`);
 /**
  *
  * @param {Client} client
@@ -17,7 +64,7 @@ import { db, pool } from './db.js';
  * @return {Promise<void>}
  */
 export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15;
     // Build a sanitized, typed data object to reduce drift and ensure defaults in one place
     const eventData = {
         action_id: (_a = event.action_id) !== null && _a !== void 0 ? _a : '',
@@ -67,7 +114,8 @@ export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function
         visitorid: (_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null,
         referrertype: (_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null,
         referrername: (_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null,
-        resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null
+        resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null,
+        experiments: (_15 = event.experiments) !== null && _15 !== void 0 ? _15 : null
     };
     // Minimal runtime validation for required fields
     if (!eventData.action_id || eventData.action_id.trim().length === 0) {
@@ -81,53 +129,26 @@ export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function
         if (!pool || typeof pool.connect !== 'function') {
             throw new Error('Database connection pool is invalid or undefined');
         }
-        // Keep the stored procedure but centralize mapping to avoid parameter mis-ordering
-        yield sql `
-      SELECT insert_into_matomo_partitioned(
-        ${eventData.action_id},
-        ${eventData.action_timestamp},
-        ${eventData.idsite},
-        ${eventData.idvisit},
-        ${eventData.actions},
-        ${eventData.country},
-        ${eventData.region},
-        ${eventData.city},
-        ${eventData.operatingsystemname},
-        ${eventData.devicemodel},
-        ${eventData.devicebrand},
-        ${eventData.visitduration},
-        ${eventData.dayssincefirstvisit},
-        ${eventData.visitortype},
-        ${eventData.sitename},
-        ${eventData.userid},
-        ${eventData.serverdateprettyfirstaction},
-        ${eventData.action_type},
-        ${eventData.action_eventcategory},
-        ${eventData.action_eventaction},
-        ${eventData.action_eventname},
-        ${eventData.action_eventvalue},
-        ${eventData.action_timespent},
-        ${eventData.usercustomproperties},
-        ${eventData.usercustomdimensions},
-        ${eventData.dimension1},
-        ${eventData.dimension2},
-        ${eventData.dimension3},
-        ${eventData.dimension4},
-        ${eventData.dimension5},
-        ${eventData.dimension6},
-        ${eventData.dimension7},
-        ${eventData.dimension8},
-        ${eventData.dimension9},
-        ${eventData.dimension10},
-        ${eventData.action_url},
-        ${eventData.sitesearchkeyword},
-        ${eventData.action_title},
-        ${eventData.visitorid},
-        ${eventData.referrertype},
-        ${eventData.referrername},
-        ${eventData.resolution}
-      )
-    `.execute(db);
+        // Use different insertion logic based on DESTINATION_TABLE
+        if (DESTINATION_TABLE === PARTITIONED_MATOMO_TABLE_NAME) {
+            // Use stored procedure for partitioned table (handles automatic partition creation)
+            yield sql `
+        SELECT insert_into_matomo_partitioned(
+          ${sql.join(MATOMO_INSERT_COLUMNS.map((column) => sql `${eventData[column]}`), sql `, `)}
+        )
+      `.execute(db);
+        }
+        else {
+            // Direct INSERT for standard (non-partitioned) table
+            yield sql `
+        INSERT INTO ${sql.id(DESTINATION_TABLE)} (
+          ${MATOMO_INSERT_COLUMN_SQL}
+        ) VALUES (
+          ${sql.join(MATOMO_INSERT_COLUMNS.map((column) => sql `${eventData[column]}`), sql `, `)}
+        )
+        ON CONFLICT (action_id) DO NOTHING
+      `.execute(db);
+        }
     }
     catch (err) {
         // Add context for troubleshooting
@@ -178,7 +199,7 @@ const actionProps = {
 };
 export const getEventsFromMatomoVisit = (matomoVisit) => {
     return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
-        var _a;
+        var _a, _b;
         const usercustomproperties = {};
         for (let k = 1; k < 10; k++) {
             const property = (_a = actionDetail.customVariables) === null || _a === void 0 ? void 0 : _a[k];
@@ -200,7 +221,7 @@ export const getEventsFromMatomoVisit = (matomoVisit) => {
             //@ts-expect-error implicit any type
             usercustomdimensions[dimension] = value;
         }
-        const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
+        const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { experiments: (_b = matomoVisit.experiments) !== null && _b !== void 0 ? _b : null, serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
             action_id: `${matomoVisit.idVisit}_${actionIndex}`
         })), { 
             // custom variables

package/dist/migrations/20260407-01-ab-testing.js

@@ -0,0 +1,343 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { sql } from 'kysely';
+const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
+const PARTITIONED_MATOMO_TABLE_NAME = process.env.PARTITIONED_MATOMO_TABLE_NAME || 'matomo_partitioned';
+export function up(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Add column for AB Testing
+        yield db.schema
+            .alterTable(MATOMO_TABLE_NAME)
+            .addColumn('experiments', 'jsonb')
+            .execute();
+        yield db.schema
+            .alterTable(PARTITIONED_MATOMO_TABLE_NAME)
+            .addColumn('experiments', 'jsonb')
+            .execute();
+        // Drop the previous version of the stored procedure so the signature can grow
+        // (CREATE OR REPLACE refuses to change parameter lists in PL/pgSQL).
+        yield sql `DROP FUNCTION IF EXISTS insert_into_matomo_partitioned(
+    text, timestamptz, text, text, text, text, text, text, text, text, text,
+    text, text, text, text, text, date, text, text, text, text, numeric, text,
+    json, json, text, text, text, text, text, text, text, text, text, text,
+    text, text, text, text, text, text, text
+  )`.execute(db);
+        yield sql `
+    CREATE OR REPLACE FUNCTION insert_into_matomo_partitioned(
+      p_action_id text,
+      p_action_timestamp timestamptz,
+      p_idsite text DEFAULT '',
+      p_idvisit text DEFAULT '',
+      p_actions text DEFAULT NULL,
+      p_country text DEFAULT NULL,
+      p_region text DEFAULT NULL,
+      p_city text DEFAULT NULL,
+      p_operatingsystemname text DEFAULT NULL,
+      p_devicemodel text DEFAULT NULL,
+      p_devicebrand text DEFAULT NULL,
+      p_visitduration text DEFAULT NULL,
+      p_dayssincefirstvisit text DEFAULT NULL,
+      p_visitortype text DEFAULT NULL,
+      p_sitename text DEFAULT NULL,
+      p_userid text DEFAULT NULL,
+      p_serverdateprettyfirstaction date DEFAULT NULL,
+      p_action_type text DEFAULT '',
+      p_action_eventcategory text DEFAULT '',
+      p_action_eventaction text DEFAULT '',
+      p_action_eventname text DEFAULT '',
+      p_action_eventvalue numeric DEFAULT 0,
+      p_action_timespent text DEFAULT '0',
+      p_usercustomproperties json DEFAULT NULL,
+      p_usercustomdimensions json DEFAULT NULL,
+      p_dimension1 text DEFAULT NULL,
+      p_dimension2 text DEFAULT NULL,
+      p_dimension3 text DEFAULT NULL,
+      p_dimension4 text DEFAULT NULL,
+      p_dimension5 text DEFAULT NULL,
+      p_dimension6 text DEFAULT NULL,
+      p_dimension7 text DEFAULT NULL,
+      p_dimension8 text DEFAULT NULL,
+      p_dimension9 text DEFAULT NULL,
+      p_dimension10 text DEFAULT NULL,
+      p_action_url text DEFAULT NULL,
+      p_sitesearchkeyword text DEFAULT NULL,
+      p_action_title text DEFAULT NULL,
+      p_visitorid text DEFAULT NULL,
+      p_referrertype text DEFAULT NULL,
+      p_referrername text DEFAULT NULL,
+      p_resolution text DEFAULT NULL,
+      p_experiments jsonb DEFAULT NULL
+    )
+    RETURNS void
+    LANGUAGE plpgsql
+    SECURITY INVOKER
+    AS $$
+    BEGIN
+        PERFORM create_weekly_partition_if_not_exists('${sql.raw(PARTITIONED_MATOMO_TABLE_NAME)}', p_action_timestamp);
+
+        INSERT INTO ${sql.id(PARTITIONED_MATOMO_TABLE_NAME)} (
+          action_id,
+          action_timestamp,
+          idsite,
+          idvisit,
+          actions,
+          country,
+          region,
+          city,
+          operatingsystemname,
+          devicemodel,
+          devicebrand,
+          visitduration,
+          dayssincefirstvisit,
+          visitortype,
+          sitename,
+          userid,
+          serverdateprettyfirstaction,
+          action_type,
+          action_eventcategory,
+          action_eventaction,
+          action_eventname,
+          action_eventvalue,
+          action_timespent,
+          usercustomproperties,
+          usercustomdimensions,
+          dimension1,
+          dimension2,
+          dimension3,
+          dimension4,
+          dimension5,
+          dimension6,
+          dimension7,
+          dimension8,
+          dimension9,
+          dimension10,
+          action_url,
+          sitesearchkeyword,
+          action_title,
+          visitorid,
+          referrertype,
+          referrername,
+          resolution,
+          experiments
+        ) VALUES (
+          p_action_id,
+          p_action_timestamp,
+          p_idsite,
+          p_idvisit,
+          p_actions,
+          p_country,
+          p_region,
+          p_city,
+          p_operatingsystemname,
+          p_devicemodel,
+          p_devicebrand,
+          p_visitduration,
+          p_dayssincefirstvisit,
+          p_visitortype,
+          p_sitename,
+          p_userid,
+          p_serverdateprettyfirstaction,
+          p_action_type,
+          p_action_eventcategory,
+          p_action_eventaction,
+          p_action_eventname,
+          p_action_eventvalue,
+          p_action_timespent,
+          p_usercustomproperties,
+          p_usercustomdimensions,
+          p_dimension1,
+          p_dimension2,
+          p_dimension3,
+          p_dimension4,
+          p_dimension5,
+          p_dimension6,
+          p_dimension7,
+          p_dimension8,
+          p_dimension9,
+          p_dimension10,
+          p_action_url,
+          p_sitesearchkeyword,
+          p_action_title,
+          p_visitorid,
+          p_referrertype,
+          p_referrername,
+          p_resolution,
+          p_experiments
+        )
+        ON CONFLICT (action_id, action_timestamp) DO NOTHING;
+    END;
+    $$;
+  `.execute(db);
+    });
+}
+export function down(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // Drop the new version of the stored procedure
+        yield sql `DROP FUNCTION IF EXISTS insert_into_matomo_partitioned(
+    text, timestamptz, text, text, text, text, text, text, text, text, text,
+    text, text, text, text, text, date, text, text, text, text, numeric, text,
+    json, json, text, text, text, text, text, text, text, text, text, text,
+    text, text, text, text, text, text, text, jsonb
+  )`.execute(db);
+        // Restore the previous version (without experiments) so the downgrade is usable
+        yield sql `
+    CREATE OR REPLACE FUNCTION insert_into_matomo_partitioned(
+      p_action_id text,
+      p_action_timestamp timestamptz,
+      p_idsite text DEFAULT '',
+      p_idvisit text DEFAULT '',
+      p_actions text DEFAULT NULL,
+      p_country text DEFAULT NULL,
+      p_region text DEFAULT NULL,
+      p_city text DEFAULT NULL,
+      p_operatingsystemname text DEFAULT NULL,
+      p_devicemodel text DEFAULT NULL,
+      p_devicebrand text DEFAULT NULL,
+      p_visitduration text DEFAULT NULL,
+      p_dayssincefirstvisit text DEFAULT NULL,
+      p_visitortype text DEFAULT NULL,
+      p_sitename text DEFAULT NULL,
+      p_userid text DEFAULT NULL,
+      p_serverdateprettyfirstaction date DEFAULT NULL,
+      p_action_type text DEFAULT '',
+      p_action_eventcategory text DEFAULT '',
+      p_action_eventaction text DEFAULT '',
+      p_action_eventname text DEFAULT '',
+      p_action_eventvalue numeric DEFAULT 0,
+      p_action_timespent text DEFAULT '0',
+      p_usercustomproperties json DEFAULT NULL,
+      p_usercustomdimensions json DEFAULT NULL,
+      p_dimension1 text DEFAULT NULL,
+      p_dimension2 text DEFAULT NULL,
+      p_dimension3 text DEFAULT NULL,
+      p_dimension4 text DEFAULT NULL,
+      p_dimension5 text DEFAULT NULL,
+      p_dimension6 text DEFAULT NULL,
+      p_dimension7 text DEFAULT NULL,
+      p_dimension8 text DEFAULT NULL,
+      p_dimension9 text DEFAULT NULL,
+      p_dimension10 text DEFAULT NULL,
+      p_action_url text DEFAULT NULL,
+      p_sitesearchkeyword text DEFAULT NULL,
+      p_action_title text DEFAULT NULL,
+      p_visitorid text DEFAULT NULL,
+      p_referrertype text DEFAULT NULL,
+      p_referrername text DEFAULT NULL,
+      p_resolution text DEFAULT NULL
+    )
+    RETURNS void
+    LANGUAGE plpgsql
+    SECURITY INVOKER
+    AS $$
+    BEGIN
+        PERFORM create_weekly_partition_if_not_exists('${sql.raw(PARTITIONED_MATOMO_TABLE_NAME)}', p_action_timestamp);
+
+        INSERT INTO ${sql.id(PARTITIONED_MATOMO_TABLE_NAME)} (
+          action_id,
+          action_timestamp,
+          idsite,
+          idvisit,
+          actions,
+          country,
+          region,
+          city,
+          operatingsystemname,
+          devicemodel,
+          devicebrand,
+          visitduration,
+          dayssincefirstvisit,
+          visitortype,
+          sitename,
+          userid,
+          serverdateprettyfirstaction,
+          action_type,
+          action_eventcategory,
+          action_eventaction,
+          action_eventname,
+          action_eventvalue,
+          action_timespent,
+          usercustomproperties,
+          usercustomdimensions,
+          dimension1,
+          dimension2,
+          dimension3,
+          dimension4,
+          dimension5,
+          dimension6,
+          dimension7,
+          dimension8,
+          dimension9,
+          dimension10,
+          action_url,
+          sitesearchkeyword,
+          action_title,
+          visitorid,
+          referrertype,
+          referrername,
+          resolution
+        ) VALUES (
+          p_action_id,
+          p_action_timestamp,
+          p_idsite,
+          p_idvisit,
+          p_actions,
+          p_country,
+          p_region,
+          p_city,
+          p_operatingsystemname,
+          p_devicemodel,
+          p_devicebrand,
+          p_visitduration,
+          p_dayssincefirstvisit,
+          p_visitortype,
+          p_sitename,
+          p_userid,
+          p_serverdateprettyfirstaction,
+          p_action_type,
+          p_action_eventcategory,
+          p_action_eventaction,
+          p_action_eventname,
+          p_action_eventvalue,
+          p_action_timespent,
+          p_usercustomproperties,
+          p_usercustomdimensions,
+          p_dimension1,
+          p_dimension2,
+          p_dimension3,
+          p_dimension4,
+          p_dimension5,
+          p_dimension6,
+          p_dimension7,
+          p_dimension8,
+          p_dimension9,
+          p_dimension10,
+          p_action_url,
+          p_sitesearchkeyword,
+          p_action_title,
+          p_visitorid,
+          p_referrertype,
+          p_referrername,
+          p_resolution
+        )
+        ON CONFLICT (action_id, action_timestamp) DO NOTHING;
+    END;
+    $$;
+  `.execute(db);
+        // Drop column for AB Testing
+        yield db.schema
+            .alterTable(MATOMO_TABLE_NAME)
+            .dropColumn('experiments')
+            .execute();
+        yield db.schema
+            .alterTable(PARTITIONED_MATOMO_TABLE_NAME)
+            .dropColumn('experiments')
+            .execute();
+    });
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@socialgouv/matomo-postgres",
   "description": "Extract visitor events from Matomo API and push to Postgres",
-  "version": "2.3.13",
+  "version": "2.4.0",
   "packageManager": "pnpm@10.28.1",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",