@socialgouv/matomo-postgres 1.5.2-beta.1 → 1.5.2-beta.3

package/dist/__tests__/importEvent.test.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+process.env.MATOMO_SITE = "42";
+process.env.PROJECT_NAME = "some-project";
+const visit_json_1 = __importDefault(require("./visit.json"));
+const importEvent_1 = require("../importEvent");
+test("getEventsFromMatomoVisit: should merge action events", () => {
+    // @ts-ignore
+    const visits = (0, importEvent_1.getEventsFromMatomoVisit)(visit_json_1.default);
+    expect(visits).toMatchSnapshot();
+});

package/dist/__tests__/index.test.js

@@ -0,0 +1,190 @@
+"use strict";
+const mock_pgQuery = jest.fn();
+const mock_matomoApi = jest.fn();
+// import formatISO from "date-fns/formatISO";
+// import addDays from "date-fns/addDays";
+// import { INITIAL_OFFSET } from "../config";
+process.env.MATOMO_SITE = "42";
+process.env.PROJECT_NAME = "some-project";
+// import matomoVisit from "./visit.json";
+// import run from "../index";
+// const NB_REQUEST_TO_INIT_DB = 3; // Number of query to init DB (createTable.js)
+// const TEST_DATE = new Date();
+// @ts-ignore
+//const isoDate = (date) => formatISO(date, { representation: "date" });
+// jest.mock("pg", () => {
+//   class Client {
+//     escapeIdentifier(name) {
+//       return name;
+//     }
+//     end() {}
+//     connect() {
+//       return Promise.resolve();
+//     }
+//     query(...args) {
+//       return Promise.resolve(mock_pgQuery(...args));
+//     }
+//   }
+//   return {
+//     Client,
+//   };
+// });
+// jest.mock("piwik-client", () => {
+//   class MatomoClient {
+//     api(...args: any[]) {
+//       return mock_matomoApi(...args);
+//     }
+//   }
+//   return MatomoClient;
+// });
+test("test", () => { });
+/*
+
+beforeEach(() => {
+  jest.resetAllMocks();
+  jest.resetModules();
+  process.env.STARTDATE = "";
+  //process.env.OFFSET = "";
+});
+
+test("run: should create table", async () => {
+  mock_pgQuery.mockReturnValue({ rows: [] });
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, []);
+  });
+  await run();
+  expect(mock_pgQuery.mock.calls[0]).toMatchSnapshot();
+});
+
+test("run: should fetch the latest event date if no date provided", async () => {
+  jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+      {
+        ...matomoVisit,
+        idVisit: 124,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
+  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
+
+  // check db queries
+  expect(mock_pgQuery.mock.calls[NB_REQUEST_TO_INIT_DB][0]).toEqual(
+    "select action_timestamp from matomo order by action_timestamp desc limit 1"
+  );
+});
+
+test("run: should resume using latest event date - offset if no date provided", async () => {
+  jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+
+  const LAST_EVENT_DATE_OFFSET = 2;
+  // @ts-ignore
+  const LAST_EVENT_DATE = addDays(TEST_DATE, -LAST_EVENT_DATE_OFFSET);
+
+  mock_pgQuery.mockReturnValue({ rows: [{ action_timestamp: LAST_EVENT_DATE.getTime() }] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+      {
+        ...matomoVisit,
+        idVisit: 124,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(
+    // @ts-ignore
+    isoDate(addDays(LAST_EVENT_DATE, -parseInt(INITIAL_OFFSET)))
+  );
+  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
+
+  const daysCount = LAST_EVENT_DATE_OFFSET + parseInt(INITIAL_OFFSET) + 1;
+  expect(mock_matomoApi.mock.calls.length).toEqual(daysCount);
+
+  // check db queries
+  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + daysCount * 7); // NB_REQUEST_TO_INIT_DB + select queries + days offset
+});
+
+test("run: should use today date if nothing in DB", async () => {
+  jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls.length).toEqual(1);
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
+
+  // check the 4 events inserted
+  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 5); // NB_REQUEST_TO_INIT_DB + check date + latest + 2 inserts
+});
+
+test("run: should use given date if any", async () => {
+  jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  // @ts-ignore
+  await run(isoDate(addDays(TEST_DATE, -10)) + "T00:00:00.000Z");
+
+  expect(mock_matomoApi.mock.calls.length).toEqual(11);
+  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 11 * 4); // NB_REQUEST_TO_INIT_DB + inserts. no initial select as date is provided
+});
+
+test("run: should use STARTDATE if any", async () => {
+  // @ts-ignore
+  process.env.STARTDATE = isoDate(addDays(TEST_DATE, -5)) + "T00:00:00.000Z";
+  jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  await run();
+
+  expect(mock_matomoApi.mock.calls.length).toEqual(6);
+
+  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + 6 * 4); // NB_REQUEST_TO_INIT_DB + initial select + inserts.
+});
+*/

package/dist/__tests__/visit.json

@@ -0,0 +1,103 @@
+{
+    "idVisit": "124",
+    "idSite": "42",
+    "country": "Argentine",
+    "operatingSystemName": "Mac",
+    "deviceBrand": "Inconnu",
+    "deviceModel": "Générique Bureau",
+    "visitDuration": "300",
+    "daysSinceFirstVisit": "23",
+    "actions": "2",
+    "visitorType": "returningCustomer",
+    "visitorId": "visitorId",
+    "referrerType": "referrerType",
+    "referrerName": "referrerName",
+    "siteName": "tests",
+    "userId": "24",
+    "region": "Buenos Aires",
+    "city": "Buenos Aires",
+    "dimension1": "guest",
+    "dimension3": "page",
+    "dimension6": "shop",
+    "dimension7": "v1.2.3",
+    "dimension8": "fr",
+    "dimension9": "light",
+    "dimension10": "36",
+    "firstActionTimestamp": 1629496512,
+    "actionDetails": [
+        {
+            "type": "event",
+            "url": "https://dive-shop.net/products/basic-wetsuit/",
+            "pageIdAction": "304",
+            "idpageview": "",
+            "serverTimePretty": "20 août 2021 21:35:18",
+            "pageId": "19696671",
+            "eventCategory": "Ecommerce",
+            "eventAction": "Cart change",
+            "timeSpent": 48,
+            "timeSpentPretty": "48s",
+            "pageviewPosition": "8",
+            "timestamp": 1629495318,
+            "icon": "plugins/Morpheus/images/event.png",
+            "iconSVG": "plugins/Morpheus/images/event.svg",
+            "title": "Evènement",
+            "subtitle": "Catégorie: \"Ecommerce', Action: \"Cart change\"",
+            "eventName": "added - Basic Wetsuit",
+            "eventValue": 1,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving",
+            "customVariables": {
+                "1": {
+                    "customVariableName1": "page-author",
+                    "customVariableValue1": "Julien"
+                },
+                "2": {
+                    "customVariableName2": "post-age",
+                    "customVariableValue2": "-430 days"
+                }
+            }
+        },
+        {
+            "type": "action",
+            "url": "https://dive-shop.net/products/diving-boots/",
+            "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
+            "pageIdAction": "60",
+            "idpageview": "8CDIez",
+            "serverTimePretty": "20 août 2021 21:30:25",
+            "pageId": "19696664",
+            "timeSpent": "2",
+            "timeSpentPretty": "2s",
+            "pageviewPosition": "5",
+            "title": "Divezone Brand Diving Boots - Divezone Store",
+            "subtitle": "https://dive-shop.net/products/diving-boots/",
+            "icon": "",
+            "iconSVG": "plugins/Morpheus/images/action.svg",
+            "timestamp": 1629495025,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving"
+        },
+        {
+            "type": "search",
+            "url": "https://dive-shop.net/products/diving-boots/",
+            "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
+            "pageIdAction": "60",
+            "idpageview": "8CDIez",
+            "serverTimePretty": "20 août 2021 21:30:25",
+            "pageId": "19696664",
+            "timeSpent": "2",
+            "timeSpentPretty": "2s",
+            "pageviewPosition": "5",
+            "title": "Divezone Brand Diving Boots - Divezone Store",
+            "subtitle": "https://dive-shop.net/products/diving-boots/",
+            "icon": "",
+            "iconSVG": "plugins/Morpheus/images/action.svg",
+            "timestamp": 1629495022,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving",
+            "siteSearchKeyword": "scuba"
+        }
+    ]
+}

package/dist/config.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RESULTPERPAGE = exports.INITIAL_OFFSET = exports.DESTINATION_TABLE = exports.PGDATABASE = exports.MATOMO_SITE = exports.MATOMO_URL = exports.MATOMO_KEY = void 0;
+exports.MATOMO_KEY = process.env.MATOMO_KEY || "";
+exports.MATOMO_URL = process.env.MATOMO_URL || "https://matomo.fabrique.social.gouv.fr/";
+exports.MATOMO_SITE = process.env.MATOMO_SITE || 0;
+exports.PGDATABASE = process.env.PGDATABASE || "";
+exports.DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+exports.INITIAL_OFFSET = process.env.INITIAL_OFFSET || "3";
+exports.RESULTPERPAGE = process.env.RESULTPERPAGE || "500";

package/dist/db.js

@@ -0,0 +1,24 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.db = void 0;
+const pg_1 = require("pg");
+const kysely_1 = require("kysely");
+const debug_1 = __importDefault(require("debug"));
+const config_1 = require("./config");
+const debug = (0, debug_1.default)("db");
+exports.db = new kysely_1.Kysely({
+    dialect: new kysely_1.PostgresDialect({
+        pool: new pg_1.Pool({
+            connectionString: config_1.PGDATABASE,
+        }),
+    }),
+    log(event) {
+        if (event.level === "query") {
+            // debug(event.query.sql);
+            // debug(event.query.parameters);
+        }
+    },
+});

package/dist/importDate.js

@@ -0,0 +1,88 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.importDate = void 0;
+const p_all_1 = __importDefault(require("p-all"));
+const debug_1 = __importDefault(require("debug"));
+const formatISO_1 = __importDefault(require("date-fns/formatISO"));
+const kysely_1 = require("kysely");
+const db_1 = require("./db");
+const importEvent_1 = require("./importEvent");
+const config_1 = require("./config");
+const debug = (0, debug_1.default)("importDate");
+/** return date as ISO yyyy-mm-dd */
+const isoDate = (date) => (0, formatISO_1.default)(date, { representation: "date" });
+/** check how many visits complete for a given date */
+const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
+    const result = yield db_1.db
+        .selectFrom(config_1.DESTINATION_TABLE)
+        .select(db_1.db.fn.count("idvisit").distinct().as("count"))
+        // UTC to be iso with matomo matomo data
+        .where((0, kysely_1.sql) `date(timezone('UTC', action_timestamp))`, "=", date)
+        .executeTakeFirst();
+    // start at previous visit in case action didnt finished to record
+    const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
+    return count;
+});
+/** import all event from givent date */
+const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void 0, void 0, function* () {
+    const limit = parseInt(config_1.RESULTPERPAGE);
+    const offset = filterOffset || (yield getRecordsCount(isoDate(date)));
+    if (!offset) {
+        debug(`${isoDate(date)}: load ${limit} visits`);
+    }
+    else {
+        debug(`${isoDate(date)}: load ${limit} more visits after ${offset}`);
+    }
+    // fetch visits details
+    const visits = yield new Promise((resolve) => piwikApi({
+        method: "Live.getLastVisitsDetails",
+        period: "day",
+        date: isoDate(date),
+        // minTimestamp: isoDate(new Date()) === isoDate(date) ? date.getTime() / 1000 : undefined, // if today, dont go further (??)
+        filter_limit: limit,
+        filter_offset: offset,
+        filter_sort_order: "asc",
+        idSite: config_1.MATOMO_SITE,
+    }, (err, visits = []) => {
+        if (err) {
+            console.error("err", err);
+            resolve([]);
+        }
+        return resolve(visits);
+    }));
+    debug(`fetched ${visits.length} visits`);
+    // flatten all events
+    const eventsFromVisits = visits.flatMap(importEvent_1.getEventsFromMatomoVisit);
+    const allEvents = eventsFromVisits.filter((event) => {
+        return true;
+    });
+    if (!allEvents.length) {
+        debug(`no more valid events after ${isoDate(date)}`);
+        return [];
+    }
+    debug(`import ${allEvents.length} events`);
+    // serial-import events into PG
+    const importedEvents = yield (0, p_all_1.default)(allEvents.map((event) => () => (0, importEvent_1.importEvent)(event)), { concurrency: 10, stopOnError: true });
+    // continue to next page if necessary
+    if (visits.length === limit) {
+        const nextOffset = offset + limit;
+        const nextEvents = yield (0, exports.importDate)(piwikApi, date, nextOffset);
+        return [...importedEvents, ...(nextEvents || [])];
+    }
+    debug(`finished importing ${isoDate(date)}, offset ${offset}`);
+    return importedEvents || [];
+});
+exports.importDate = importDate;
+module.exports = { importDate: exports.importDate };

package/dist/importEvent.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getEventsFromMatomoVisit = exports.importEvent = void 0;
+const config_1 = require("./config");
+const db_1 = require("./db");
+/**
+ *
+ * @param {Client} client
+ * @param {import("types").Event} event
+ *
+ * @return {Promise<Record<"rows", any[]>>}
+ */
+const importEvent = (event) => 
+// @ts-ignore // TODO
+db_1.db
+    .insertInto(config_1.DESTINATION_TABLE)
+    .values([Object.assign({}, event)])
+    .onConflict((oc) => oc.doNothing())
+    .execute();
+exports.importEvent = importEvent;
+const matomoProps = [
+    "idSite",
+    "idVisit",
+    "actions",
+    "country",
+    "region",
+    "city",
+    "operatingSystemName",
+    "deviceModel",
+    "deviceBrand",
+    "visitDuration",
+    "daysSinceFirstVisit",
+    "visitorType",
+    "visitorId",
+    "referrerType",
+    "referrerName",
+    "siteName",
+    "userId",
+];
+/** @type Record<string, (a: import("types/matomo-api").ActionDetail) => string | number> */
+const actionProps = {
+    action_type: (action) => action.type,
+    action_title: (action) => action.title,
+    action_eventcategory: (action) => action.eventCategory,
+    action_eventaction: (action) => action.eventAction,
+    action_eventname: (action) => action.eventName,
+    action_eventvalue: (action) => action.eventValue,
+    action_timespent: (action) => action.timeSpent,
+    action_timestamp: (action) => new Date(action.timestamp * 1000).toISOString(),
+    action_url: (action) => action.url,
+    sitesearchkeyword: (action) => action.siteSearchKeyword,
+};
+const getEventsFromMatomoVisit = (matomoVisit) => {
+    return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
+        const usercustomproperties = {};
+        for (let k = 1; k < 10; k++) {
+            const property = actionDetail.customVariables && actionDetail.customVariables[k];
+            if (!property)
+                continue; // max 10 custom variables
+            //@ts-ignore
+            usercustomproperties[property[`customVariableName${k}`]] = property[`customVariableValue${k}`];
+        }
+        /** @type {Record<string, string>} */
+        const usercustomdimensions = {};
+        for (let k = 1; k < 11; k++) {
+            const dimension = `dimension${k}`;
+            //@ts-ignore
+            const value = actionDetail[dimension] || matomoVisit[dimension];
+            if (!value)
+                continue; // max 10 custom variables
+            //@ts-ignore
+            usercustomdimensions[dimension] = value;
+        }
+        const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
+            action_id: `${matomoVisit.idVisit}_${actionIndex}`,
+        })), { 
+            // custom variables
+            usercustomproperties,
+            // custom dimensions
+            // We keep both for backwards compatibility.
+            // Current implementation is flat with one column for each dimension.
+            usercustomdimensions }), usercustomdimensions);
+        return event;
+    });
+};
+exports.getEventsFromMatomoVisit = getEventsFromMatomoVisit;

package/dist/index.js

@@ -0,0 +1,82 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kysely_1 = require("kysely");
+const p_all_1 = __importDefault(require("p-all"));
+const debug_1 = __importDefault(require("debug"));
+const eachDayOfInterval_1 = __importDefault(require("date-fns/eachDayOfInterval"));
+const piwik_client_1 = __importDefault(require("piwik-client"));
+const db_1 = require("./db");
+const config_1 = require("./config");
+const importDate_1 = require("./importDate");
+const debug = (0, debug_1.default)("index");
+function run(date) {
+    return __awaiter(this, void 0, void 0, function* () {
+        debug("run, date=" + date);
+        const piwik = new piwik_client_1.default(config_1.MATOMO_URL, config_1.MATOMO_KEY);
+        // priority:
+        //  - optional parameter date
+        //  - last event in the table
+        //  - optional env.STARTDATE
+        //  - today
+        let referenceDate;
+        if (!referenceDate && date)
+            referenceDate = new Date(date);
+        if (!referenceDate)
+            referenceDate = yield findLastEventInMatomo(db_1.db);
+        if (!referenceDate && process.env.STARTDATE)
+            referenceDate = new Date(process.env.STARTDATE);
+        if (!referenceDate)
+            referenceDate = new Date(new Date().getTime() - +config_1.INITIAL_OFFSET * 24 * 60 * 60 * 1000);
+        const dates = (0, eachDayOfInterval_1.default)({
+            start: referenceDate,
+            end: new Date(new Date().getTime()),
+        });
+        debug(`import starting at : ${dates[0].toISOString()}`);
+        // for each date, serial-import data
+        const res = yield (0, p_all_1.default)(dates.map((date) => () => (0, importDate_1.importDate)(piwik.api.bind(piwik), date)), { concurrency: 1, stopOnError: true });
+        debug("close");
+        return res;
+    });
+}
+exports.default = run;
+if (require.main === module) {
+    (() => __awaiter(void 0, void 0, void 0, function* () {
+        if (!config_1.MATOMO_SITE)
+            return console.error("Missing env MATOMO_SITE");
+        if (!config_1.MATOMO_KEY)
+            return console.error("Missing env MATOMO_KEY");
+        if (!config_1.PGDATABASE)
+            return console.error("Missing env PGDATABASE");
+        yield run();
+        debug("run finished");
+        db_1.db.destroy();
+    }))();
+}
+function findLastEventInMatomo(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const latest = yield db
+            .selectFrom(config_1.DESTINATION_TABLE)
+            .select((0, kysely_1.sql) `action_timestamp at time zone 'UTC'`.as("action_timestamp"))
+            .orderBy("action_timestamp", "desc")
+            .limit(1)
+            .executeTakeFirst();
+        if (latest) {
+            // check from the day before just to be sure we haev all events
+            const date = new Date(new Date(latest.action_timestamp).getTime() - 2 * 24 * 60 * 60 * 1000);
+            return date;
+        }
+        return null;
+    });
+}

package/dist/migrate-down.js

@@ -0,0 +1,66 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const fs_1 = require("fs");
+const kysely_1 = require("kysely");
+const db_1 = require("./db");
+function migrateDown() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const migrator = new kysely_1.Migrator({
+            db: db_1.db,
+            provider: new kysely_1.FileMigrationProvider({
+                fs: fs_1.promises,
+                path,
+                migrationFolder: __dirname + "/migrations",
+            }),
+        });
+        const { error, results } = yield migrator.migrateDown();
+        results === null || results === void 0 ? void 0 : results.forEach((it) => {
+            if (it.status === "Success") {
+                console.log(`down migration "${it.migrationName}" was executed successfully`);
+            }
+            else if (it.status === "Error") {
+                console.error(`failed to execute down migration "${it.migrationName}"`);
+            }
+        });
+        if (error) {
+            console.error("failed to down migrate");
+            console.error(error);
+            process.exit(1);
+        }
+        yield db_1.db.destroy();
+    });
+}
+migrateDown();

package/dist/migrate-latest.js

@@ -0,0 +1,86 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const path = __importStar(require("path"));
+const fs_1 = require("fs");
+const kysely_1 = require("kysely");
+const db_1 = require("./db");
+const config_1 = require("./config");
+function migrateToLatest() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const extension = yield db_1.db
+            .selectFrom("pg_extension")
+            //@ts-ignore
+            .select("extname")
+            //@ts-ignore
+            .where("extname", "=", "pg_partman")
+            .executeTakeFirst();
+        if (extension) {
+            console.error("pg_partman extension detected; Skip migrations");
+            return;
+        }
+        const migrator = new kysely_1.Migrator({
+            db: db_1.db,
+            provider: new kysely_1.FileMigrationProvider({
+                fs: fs_1.promises,
+                path,
+                migrationFolder: __dirname + "/migrations",
+            }),
+            // allow to have mutliple migratable instances in a single schema
+            migrationTableName: `${config_1.DESTINATION_TABLE}_migration`,
+            migrationLockTableName: `${config_1.DESTINATION_TABLE}_migration_lock`,
+        });
+        const { error, results } = yield migrator.migrateToLatest();
+        results === null || results === void 0 ? void 0 : results.forEach((it) => {
+            if (it.status === "Success") {
+                console.log(`migration "${it.migrationName}" was executed successfully`);
+            }
+            else if (it.status === "Error") {
+                console.error(`failed to execute migration "${it.migrationName}"`);
+            }
+        });
+        if (error) {
+            console.error("failed to migrate");
+            console.error(error);
+            process.exit(1);
+        }
+        else {
+            if (!(results === null || results === void 0 ? void 0 : results.length)) {
+                console.log("No migration to run");
+            }
+        }
+        yield db_1.db.destroy();
+    });
+}
+migrateToLatest();

package/dist/migrations/20230301-01-initial.js

@@ -0,0 +1,70 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const kysely_1 = require("kysely");
+const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+function up(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield db.schema
+            .createTable(DESTINATION_TABLE)
+            .ifNotExists()
+            .addColumn("action_id", "text", (col) => col.unique().notNull())
+            .addColumn("idsite", "text")
+            .addColumn("idvisit", "text")
+            .addColumn("actions", "text")
+            .addColumn("country", "text")
+            .addColumn("region", "text")
+            .addColumn("city", "text")
+            .addColumn("operatingsystemname", "text")
+            .addColumn("devicemodel", "text")
+            .addColumn("devicebrand", "text")
+            .addColumn("visitduration", "text")
+            .addColumn("dayssincefirstvisit", "text")
+            .addColumn("visitortype", "text")
+            .addColumn("sitename", "text")
+            .addColumn("userid", "text")
+            .addColumn("serverdateprettyfirstaction", "date")
+            .addColumn("action_type", "text")
+            .addColumn("action_eventcategory", "text")
+            .addColumn("action_eventaction", "text")
+            .addColumn("action_eventname", "text")
+            .addColumn("action_eventvalue", "numeric")
+            .addColumn("action_timespent", "text")
+            .addColumn("action_timestamp", "timestamptz", (col) => col.defaultTo((0, kysely_1.sql) `now()`))
+            .addColumn("usercustomproperties", "json")
+            .addColumn("usercustomdimensions", "json")
+            .addColumn("dimension1", "text")
+            .addColumn("dimension2", "text")
+            .addColumn("dimension3", "text")
+            .addColumn("dimension4", "text")
+            .addColumn("dimension5", "text")
+            .addColumn("dimension6", "text")
+            .addColumn("dimension7", "text")
+            .addColumn("dimension8", "text")
+            .addColumn("dimension9", "text")
+            .addColumn("dimension10", "text")
+            .addColumn("action_url", "text")
+            .addColumn("sitesearchkeyword", "text")
+            .addColumn("action_title", "text")
+            .addColumn("visitorid", "text")
+            .addColumn("referrertype", "text")
+            .addColumn("referrername", "text")
+            .execute();
+    });
+}
+exports.up = up;
+function down(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield db.schema.dropTable(DESTINATION_TABLE).execute();
+    });
+}
+exports.down = down;

package/dist/migrations/20230301-02-indexes.js

@@ -0,0 +1,105 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.down = exports.up = void 0;
+const kysely_1 = require("kysely");
+const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+const indexes = [
+    {
+        name: "idx_action_eventaction_matomo",
+        columns: ["action_eventaction"],
+    },
+    {
+        name: "idx_action_eventcategory_matomo",
+        columns: ["action_eventcategory"],
+    },
+    {
+        name: "idx_action_id",
+        columns: ["action_id"],
+    },
+    {
+        name: "idx_action_timestamp_matomo",
+        columns: ["action_timestamp"],
+    },
+    {
+        name: "idx_action_type_matomo",
+        columns: ["action_type"],
+    },
+    {
+        name: "idx_actionurl",
+        columns: ["action_url"],
+    },
+    {
+        name: "idx_dimension1",
+        columns: ["dimension1"],
+    },
+    {
+        name: "idx_dimension2",
+        columns: ["dimension2"],
+    },
+    {
+        name: "idx_dimension3",
+        columns: ["dimension3"],
+    },
+    {
+        name: "idx_dimension4",
+        columns: ["dimension4"],
+    },
+    {
+        name: "idx_dimension5",
+        columns: ["dimension5"],
+    },
+    {
+        name: "idx_idvisit_matomo",
+        columns: ["idvisit"],
+    },
+    {
+        name: "idx_region",
+        columns: ["region"],
+    },
+    {
+        name: "idx_userid",
+        columns: ["userid"],
+    },
+    {
+        name: "idx_visitorid",
+        columns: ["visitorid"],
+    },
+];
+function up(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        indexes.forEach((index) => __awaiter(this, void 0, void 0, function* () {
+            yield db.schema
+                .createIndex(index.name)
+                .ifNotExists()
+                .on(DESTINATION_TABLE)
+                .using("btree")
+                .columns(index.columns)
+                .execute();
+        }));
+        yield db.schema
+            .createIndex("actions_day")
+            .ifNotExists()
+            .on(DESTINATION_TABLE)
+            .expression((0, kysely_1.sql) `date(timezone('UTC', action_timestamp))`)
+            .execute();
+    });
+}
+exports.up = up;
+function down(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        indexes.forEach((index) => __awaiter(this, void 0, void 0, function* () {
+            yield db.schema.dropIndex(index.name).execute();
+        }));
+        db.schema.dropIndex("actions_day").execute();
+    });
+}
+exports.down = down;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@socialgouv/matomo-postgres",
   "description": "Extract visitor events from Matomo API and push to Postgres",
-  "version": "1.5.2-beta.1",
+  "version": "1.5.2-beta.3",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",
   "main": "dist/index.js",