@socialgouv/matomo-postgres 1.1.4

package/src/__tests__/importDate.test.js

@@ -0,0 +1,133 @@
+const { importDate } = require("../importDate");
+
+const mock_pgQuery = jest.fn();
+const mock_matomoApi = jest.fn();
+
+process.env.MATOMO_SITE = "42";
+process.env.PROJECT_NAME = "some-project";
+
+const matomoVisit = require("./visit.json");
+
+jest.mock("pg", () => {
+  class Client {
+    escapeIdentifier(name) {
+      return name;
+    }
+    end() {}
+    connect() {
+      return Promise.resolve();
+    }
+    query(...args) {
+      return mock_pgQuery(...args);
+    }
+  }
+  return {
+    Client,
+  };
+});
+
+jest.mock("piwik-client", () => {
+  class MatomoClient {
+    api(...args) {
+      return mock_matomoApi(...args);
+    }
+  }
+  return MatomoClient;
+});
+
+beforeEach(() => {
+  jest.resetAllMocks();
+  jest.resetModules();
+  process.env.STARTDATE = "";
+});
+
+test("importDate: should import all events for a given date", async () => {
+  const pgSpy = jest.fn().mockResolvedValue();
+  const matomoSpy = jest.fn().mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+      {
+        ...matomoVisit,
+        idVisit: 124,
+      },
+    ]);
+  });
+  const fakeClient = {
+    escapeIdentifier(name) {
+      return name;
+    },
+    query: pgSpy,
+  };
+  //@ts-expect-error
+  await importDate(fakeClient, matomoSpy, new Date("2021-08-02"));
+  expect(matomoSpy.mock.calls.map((c) => c[0])).toMatchSnapshot();
+  expect(pgSpy.mock.calls).toMatchSnapshot();
+});
+
+test("importDate: should import when new results", async () => {
+  const pgResult = { rows: [] };
+  const pgSpy = jest.fn().mockResolvedValue(pgResult);
+  const matomoSpy = jest.fn().mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+  const fakeClient = {
+    query: pgSpy,
+    escapeIdentifier(name) {
+      return name;
+    },
+  };
+  //@ts-expect-error
+  const imported = await importDate(fakeClient, matomoSpy, new Date("2021-08-03T00:00:00"));
+  expect(pgSpy.mock.calls.length).toEqual(3);
+  expect(imported.length).toEqual(2);
+  expect(matomoSpy.mock.calls[0][0].filter_offset).toEqual(0);
+});
+
+test("importDate: should offset matomo calls when results already exist", async () => {
+  const pgResult = { rows: [{ count: 42 }] };
+  const pgSpy = jest.fn().mockResolvedValue(pgResult);
+  const matomoSpy = jest.fn().mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+  const fakeClient = {
+    query: pgSpy,
+    escapeIdentifier(name) {
+      return name;
+    },
+  };
+  //@ts-expect-error
+  const imported = await importDate(fakeClient, matomoSpy, new Date("2021-08-03T00:00:00"));
+  expect(pgSpy.mock.calls.length).toEqual(3);
+  expect(imported.length).toEqual(2);
+  expect(matomoSpy.mock.calls[0][0].filter_offset).toEqual(42);
+});
+
+test("importDate: should NOT import when no matomo results", async () => {
+  const pgSpy = jest.fn().mockResolvedValue({ rows: [{ action_timestamp: 1627948800000 }] });
+  const matomoSpy = jest.fn().mockImplementation((options, cb) => {
+    return cb(null, []);
+  });
+  const fakeClient = {
+    query: pgSpy,
+    escapeIdentifier(name) {
+      return name;
+    },
+  };
+  //@ts-expect-error
+  const imported = await importDate(fakeClient, matomoSpy, new Date("2021-08-03"));
+  expect(pgSpy.mock.calls.length).toEqual(1);
+  expect(imported.length).toEqual(0);
+});

package/src/__tests__/importEvent.test.js

@@ -0,0 +1,22 @@
+process.env.MATOMO_SITE = "42";
+process.env.PROJECT_NAME = "some-project";
+
+const matomoVisit = require("./visit.json");
+const { getEventsFromMatomoVisit, importEvent } = require("../importEvent");
+
+test("importEvent: should extract events from matomo visit actionsDetails and create insert queries", () => {
+  //@ts-expect-error
+  const visits = getEventsFromMatomoVisit(matomoVisit);
+  expect(visits).toMatchSnapshot();
+  const spy = jest.fn();
+  spy.mockResolvedValue(Promise.resolve());
+  const fakeClient = {
+    escapeIdentifier(name) {
+      return name;
+    },
+    query: spy,
+  };
+  //@ts-expect-error
+  visits.map((visit) => importEvent(fakeClient, visit));
+  expect(spy.mock.calls).toMatchSnapshot();
+});

package/src/__tests__/index.test.js

@@ -0,0 +1,198 @@
+const mock_pgQuery = jest.fn();
+const mock_matomoApi = jest.fn();
+const formatISO = require("date-fns/formatISO");
+const addDays = require("date-fns/addDays");
+
+const { OFFSET } = require("../config");
+
+process.env.MATOMO_SITE = "42";
+process.env.PROJECT_NAME = "some-project";
+
+const matomoVisit = require("./visit.json");
+
+const run = require("../index");
+
+const TEST_DATE = new Date();
+
+// @ts-ignore
+const isoDate = (date) => formatISO(date, { representation: "date" });
+
+jest.mock("pg", () => {
+  class Client {
+    escapeIdentifier(name) {
+      return name;
+    }
+    end() {}
+    connect() {
+      return Promise.resolve();
+    }
+    query(...args) {
+      return Promise.resolve(mock_pgQuery(...args));
+    }
+  }
+  return {
+    Client,
+  };
+});
+
+jest.mock("piwik-client", () => {
+  class MatomoClient {
+    api(...args) {
+      return mock_matomoApi(...args);
+    }
+  }
+  return MatomoClient;
+});
+
+beforeEach(() => {
+  jest.resetAllMocks();
+  jest.resetModules();
+  process.env.STARTDATE = "";
+  //process.env.OFFSET = "";
+});
+
+test("run: should create table", async () => {
+  mock_pgQuery.mockReturnValue({ rows: [] });
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, []);
+  });
+  await run();
+  expect(mock_pgQuery.mock.calls[0]).toMatchSnapshot();
+});
+
+test("run: should fetch the latest event date if no date provided", async () => {
+  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+      {
+        ...matomoVisit,
+        idVisit: 124,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
+  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
+
+  // check db queries
+  expect(mock_pgQuery.mock.calls[3][0]).toEqual(
+    // call 0 is create table
+    // call 1 is add column usercustomdimension
+    // call 2 is add column action_url
+    //
+    "select action_timestamp from matomo order by action_timestamp desc limit 1"
+  );
+});
+
+test("run: should resume using latest event date - offset if no date provided", async () => {
+  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
+
+  const LAST_EVENT_DATE_OFFSET = 2;
+  // @ts-ignore
+  const LAST_EVENT_DATE = addDays(TEST_DATE, -LAST_EVENT_DATE_OFFSET);
+
+  mock_pgQuery.mockReturnValue({ rows: [{ action_timestamp: LAST_EVENT_DATE.getTime() }] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+      {
+        ...matomoVisit,
+        idVisit: 124,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(
+    // @ts-ignore
+    isoDate(addDays(LAST_EVENT_DATE, -parseInt(OFFSET)))
+  );
+  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
+
+  const daysCount = LAST_EVENT_DATE_OFFSET + parseInt(OFFSET) + 1;
+  expect(mock_matomoApi.mock.calls.length).toEqual(daysCount);
+
+  // check db queries
+  expect(mock_pgQuery.mock.calls.length).toEqual(4 + daysCount * 5); // create + alter + alter + select queries + days offset
+});
+
+test("run: should use today date if nothing in DB", async () => {
+  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  await run();
+
+  // check matomo requests
+  expect(mock_matomoApi.mock.calls.length).toEqual(1);
+  console.log(TEST_DATE, isoDate(TEST_DATE));
+  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
+
+  // check the 4 events inserted
+  expect(mock_pgQuery.mock.calls.length).toEqual(4 + 3); // create, alter, alter, check date, latest + 2 inserts
+});
+
+test("run: should use given date if any", async () => {
+  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  // @ts-ignore
+  await run(isoDate(addDays(TEST_DATE, -10)) + "T00:00:00.000Z");
+
+  expect(mock_matomoApi.mock.calls.length).toEqual(11);
+  expect(mock_pgQuery.mock.calls.length).toEqual(3 + 11 * 3); // create table + alter + alter + inserts. no initial select as date is provided
+});
+
+test("run: should use STARTDATE if any", async () => {
+  // @ts-ignore
+  process.env.STARTDATE = isoDate(addDays(TEST_DATE, -5)) + "T00:00:00.000Z";
+  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
+  mock_pgQuery.mockReturnValue({ rows: [] });
+
+  mock_matomoApi.mockImplementation((options, cb) => {
+    return cb(null, [
+      {
+        ...matomoVisit,
+        idVisit: 123,
+      },
+    ]);
+  });
+
+  await run();
+
+  expect(mock_matomoApi.mock.calls.length).toEqual(6);
+
+  expect(mock_pgQuery.mock.calls.length).toEqual(4 + 6 * 3); // create table + alter + alter + initial select + inserts.
+});

package/src/__tests__/visit.json

@@ -0,0 +1,74 @@
+{
+  "idVisit": "124",
+  "idSite": "42",
+  "country": "Argentine",
+  "operatingSystemName": "Mac",
+  "deviceBrand": "Inconnu",
+  "deviceModel": "Générique Bureau",
+  "visitDuration": "300",
+  "daysSinceFirstVisit": "23",
+  "actions": "2",
+  "visitorType": "returningCustomer",
+  "siteName": "tests",
+  "userId": "24",
+  "region": "Buenos Aires",
+  "city": "Buenos Aires",
+  "dimension1": "guest",
+  "dimension3": "page",
+  "firstActionTimestamp": 1629496512,
+  "actionDetails": [
+    {
+      "type": "event",
+      "url": "https://dive-shop.net/products/basic-wetsuit/",
+      "pageIdAction": "304",
+      "idpageview": "",
+      "serverTimePretty": "20 août 2021 21:35:18",
+      "pageId": "19696671",
+      "eventCategory": "Ecommerce",
+      "eventAction": "Cart change",
+      "timeSpent": 48,
+      "timeSpentPretty": "48s",
+      "pageviewPosition": "8",
+      "timestamp": 1629495318,
+      "icon": "plugins/Morpheus/images/event.png",
+      "iconSVG": "plugins/Morpheus/images/event.svg",
+      "title": "Evènement",
+      "subtitle": "Catégorie: \"Ecommerce', Action: \"Cart change\"",
+      "eventName": "added - Basic Wetsuit",
+      "eventValue": 1,
+      "dimension2": "julien",
+      "dimension4": "indonesia",
+      "dimension5": "diving",
+      "customVariables": {
+        "1": {
+          "customVariableName1": "page-author",
+          "customVariableValue1": "Julien"
+        },
+        "2": {
+          "customVariableName2": "post-age",
+          "customVariableValue2": "-430 days"
+        }
+      }
+    },
+    {
+      "type": "action",
+      "url": "https://dive-shop.net/products/diving-boots/",
+      "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
+      "pageIdAction": "60",
+      "idpageview": "8CDIez",
+      "serverTimePretty": "20 août 2021 21:30:25",
+      "pageId": "19696664",
+      "timeSpent": "2",
+      "timeSpentPretty": "2s",
+      "pageviewPosition": "5",
+      "title": "Divezone Brand Diving Boots - Divezone Store",
+      "subtitle": "https://dive-shop.net/products/diving-boots/",
+      "icon": "",
+      "iconSVG": "plugins/Morpheus/images/action.svg",
+      "timestamp": 1629495025,
+      "dimension2": "julien",
+      "dimension4": "indonesia",
+      "dimension5": "diving"
+    }
+  ]
+}

package/src/config.js

@@ -0,0 +1,9 @@
+const MATOMO_KEY = process.env.MATOMO_KEY || "";
+const MATOMO_URL = process.env.MATOMO_URL || "https://matomo.fabrique.social.gouv.fr/";
+const MATOMO_SITE = process.env.MATOMO_SITE || 0;
+const PGDATABASE = process.env.PGDATABASE || "";
+const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+const OFFSET = process.env.OFFSET || "3";
+const RESULTPERPAGE = process.env.RESULTPERPAGE || "500";
+
+module.exports = { MATOMO_KEY, MATOMO_URL, MATOMO_SITE, PGDATABASE, DESTINATION_TABLE, OFFSET, RESULTPERPAGE };

package/src/createTable.js

@@ -0,0 +1,48 @@
+const { Client } = require("pg");
+
+const { DESTINATION_TABLE } = require("./config");
+/**
+ *
+ * @param {Client} client
+ */
+async function createTable(client) {
+  const text = `CREATE TABLE IF NOT EXISTS ${client.escapeIdentifier(DESTINATION_TABLE)}
+  (
+    idsite                      text,
+    idvisit                     text,
+    actions                     text,
+    country                     text,
+    region                      text,
+    city                        text,
+    operatingsystemname         text,
+    devicemodel                 text,
+    devicebrand                 text,
+    visitduration               text,
+    dayssincefirstvisit         text,
+    visitortype                 text,
+    sitename                    text,
+    userid                      text,
+    serverdateprettyfirstaction date,
+    action_id                   text UNIQUE,
+    action_type                 text,
+    action_eventcategory        text,
+    action_eventaction          text,
+    action_eventname            text,
+    action_eventvalue           text,
+    action_timespent            text,
+    action_timestamp            timestamp with time zone,
+    usercustomproperties        json,
+    usercustomdimensions        json,
+    action_url                  text
+)`;
+  await client.query(text, []);
+  const addUserCustomDimensionColumn = `ALTER TABLE IF EXISTS  ${client.escapeIdentifier(DESTINATION_TABLE)} 
+  ADD COLUMN IF NOT EXISTS "usercustomdimensions" json;`;
+  await client.query(addUserCustomDimensionColumn, []);
+
+  const addActionUrlColumn = `ALTER TABLE IF EXISTS  ${client.escapeIdentifier(DESTINATION_TABLE)} 
+  ADD COLUMN IF NOT EXISTS "action_url" text;`;
+  await client.query(addActionUrlColumn, []);
+}
+
+module.exports = { createTable };

package/src/importDate.js

@@ -0,0 +1,105 @@
+const { Client } = require("pg");
+const pAll = require("p-all");
+const debug = require("debug")("importDate");
+const formatISO = require("date-fns/formatISO");
+
+const { importEvent, getEventsFromMatomoVisit } = require("./importEvent");
+
+const { RESULTPERPAGE, MATOMO_SITE, DESTINATION_TABLE } = require("./config");
+
+/**
+ * return date as ISO yyyy-mm-dd
+ *
+ * @param {Date} date date
+ *
+ * @returns {string}
+ */
+// @ts-ignore
+const isoDate = (date) => formatISO(date, { representation: "date" });
+
+/**
+ * check count if imported rows for given date
+ *
+ * @param {Client} client PG client
+ * @param {Date} date datetime as ISO string
+ *
+ * @returns {Promise<any>}
+ */
+const getRecordsCount = async (client, date) => {
+  const text = `SELECT COUNT(distinct idvisit) FROM ${client.escapeIdentifier(
+    DESTINATION_TABLE
+  )} WHERE action_timestamp::date='${isoDate(date)}';`;
+  const result = await client.query(text);
+  return parseInt((result && result.rows && result.rows.length && result.rows[0].count) || 0);
+};
+
+/**
+ * import all matomo data for a given date
+ *
+ * @param {Client} client PG client
+ * @param {any} piwikApi piwik.api instance
+ * @param {Date} date datetime as ISO string
+ *
+ * @returns {Promise<any[]>}
+ */
+const importDate = async (client, piwikApi, date, filterOffset = 0) => {
+  const limit = parseInt(RESULTPERPAGE);
+  const offset = filterOffset || (await getRecordsCount(client, date));
+  if (!offset) {
+    debug(`${isoDate(date)}: load ${limit} visits`);
+  } else {
+    debug(`${isoDate(date)}: load ${limit} more visits after ${offset}`);
+  }
+  // fetch visits details
+  const visits = await new Promise((resolve) =>
+    piwikApi(
+      {
+        method: "Live.getLastVisitsDetails",
+        period: "day",
+        date: isoDate(date),
+        filter_limit: limit,
+        filter_offset: offset,
+        filter_sort_order: "asc",
+        idSite: MATOMO_SITE,
+      },
+      (err, responseObject = []) => {
+        if (err) {
+          console.error("err", err);
+          resolve([]);
+        }
+        return resolve(responseObject || []);
+      }
+    )
+  );
+
+  debug(`fetched ${visits.length} visits`);
+
+  // flatten events
+  const allEvents = visits.flatMap(getEventsFromMatomoVisit);
+
+  if (!allEvents.length) {
+    debug(`no more valid events after ${isoDate(date)}`);
+    return [];
+  }
+
+  debug(`import ${allEvents.length} events`);
+
+  // serial-import events into PG
+  const importedEvents = await pAll(
+    allEvents.map((event) => () => importEvent(client, event)),
+    { concurrency: 10, stopOnError: true }
+  );
+
+  // continue to next page if necessary
+  if (visits.length === limit) {
+    const nextOffset = offset + limit;
+    const nextEvents = await importDate(client, piwikApi, date, nextOffset);
+    return [...importedEvents, ...(nextEvents || [])];
+  }
+
+  debug(`finished importing ${isoDate(date)}, offset ${offset}`);
+
+  return importedEvents || [];
+};
+
+module.exports = { importDate };

package/src/importEvent.js

@@ -0,0 +1,99 @@
+const { Client } = require("pg");
+
+const { DESTINATION_TABLE } = require("./config");
+
+/**
+ *
+ * @param {Client} client
+ * @param {import("types").Event} event
+ *
+ * @return {Promise<Record<"rows", any[]>>}
+ */
+const importEvent = (client, event) => {
+  const eventKeys = Object.keys(event);
+  const text = `insert into ${client.escapeIdentifier(DESTINATION_TABLE)}
+        (${eventKeys.join(", ")})
+        values (${eventKeys.map((k, i) => `\$${i + 1}`).join(", ")})
+        ON CONFLICT DO NOTHING`;
+  const values = [...eventKeys.map((k) => event[k])];
+  return client.query(text, values).catch((e) => {
+    console.log("QUERY error", e);
+    return { rows: [] };
+  });
+};
+
+const matomoProps = [
+  "idSite",
+  "idVisit",
+  "actions",
+  "country",
+  "region",
+  "city",
+  "operatingSystemName",
+  "deviceModel",
+  "deviceBrand",
+  "visitDuration",
+  "daysSinceFirstVisit",
+  "visitorType",
+  "siteName",
+  "userId",
+];
+
+/** @type Record<string, (a: import("types/matomo").ActionDetail) => string | number> */
+const actionProps = {
+  action_type: (action) => action.type,
+  action_eventcategory: (action) => action.eventCategory,
+  action_eventaction: (action) => action.eventAction,
+  action_eventname: (action) => action.eventName,
+  action_eventvalue: (action) => action.eventValue,
+  action_timespent: (action) => action.timeSpent,
+  action_timestamp: (action) => new Date(action.timestamp * 1000).toISOString(),
+  action_url: (action) => action.url,
+};
+
+/**
+ * Convert a single matomo visit {<import("types/matomo").Visit} to multiple {Event[]}
+ *
+ * @param {Partial<import("types/matomo").Visit>} matomoVisit
+ *
+ * @return {import("types").Event[]} list of events
+ *
+ */
+const getEventsFromMatomoVisit = (matomoVisit) => {
+  return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
+    /** @type {Record<string, string>} */
+    const usercustomproperties = {};
+    for (let k = 1; k < 10; k++) {
+      const property = actionDetail.customVariables && actionDetail.customVariables[k];
+      if (!property) continue; // max 10 custom variables
+      usercustomproperties[property[`customVariableName${k}`]] = property[`customVariableValue${k}`];
+    }
+
+    /** @type {Record<string, string>} */
+    const usercustomdimensions = {};
+    for (let k = 1; k < 10; k++) {
+      const dimension = `dimension${k}`;
+      const value = actionDetail[dimension] || matomoVisit[dimension];
+      if (!value) continue; // max 10 custom variables
+      usercustomdimensions[dimension] = value;
+    }
+
+    /** @type {import("types").Event} */
+    // @ts-ignore
+    const event = {
+      // default matomo visit properties
+      ...matomoProps.reduce((a, prop) => ({ ...a, [prop.toLowerCase()]: matomoVisit[prop] }), {}),
+      serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString(),
+      // action specific properties
+      ...Object.keys(actionProps).reduce((a, prop) => ({ ...a, [prop]: actionProps[prop](actionDetail) }), {
+        action_id: `${matomoVisit.idVisit}_${actionIndex}`,
+      }),
+      // custom variables
+      usercustomproperties,
+      usercustomdimensions,
+    };
+    return event;
+  });
+};
+
+module.exports = { getEventsFromMatomoVisit, importEvent };