@socialgouv/matomo-postgres 1.5.0 → 1.5.2-beta.1

package/src/__tests__/index.test.js

@@ -1,194 +0,0 @@
-const mock_pgQuery = jest.fn();
-const mock_matomoApi = jest.fn();
-const formatISO = require("date-fns/formatISO");
-const addDays = require("date-fns/addDays");
-
-const { OFFSET } = require("../config");
-
-process.env.MATOMO_SITE = "42";
-process.env.PROJECT_NAME = "some-project";
-
-const matomoVisit = require("./visit.json");
-
-const run = require("../index");
-
-const NB_REQUEST_TO_INIT_DB = 3; // Number of query to init DB (createTable.js)
-const TEST_DATE = new Date();
-
-// @ts-ignore
-const isoDate = (date) => formatISO(date, { representation: "date" });
-
-jest.mock("pg", () => {
-  class Client {
-    escapeIdentifier(name) {
-      return name;
-    }
-    end() {}
-    connect() {
-      return Promise.resolve();
-    }
-    query(...args) {
-      return Promise.resolve(mock_pgQuery(...args));
-    }
-  }
-  return {
-    Client,
-  };
-});
-
-jest.mock("piwik-client", () => {
-  class MatomoClient {
-    api(...args) {
-      return mock_matomoApi(...args);
-    }
-  }
-  return MatomoClient;
-});
-
-beforeEach(() => {
-  jest.resetAllMocks();
-  jest.resetModules();
-  process.env.STARTDATE = "";
-  //process.env.OFFSET = "";
-});
-
-test("run: should create table", async () => {
-  mock_pgQuery.mockReturnValue({ rows: [] });
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, []);
-  });
-  await run();
-  expect(mock_pgQuery.mock.calls[0]).toMatchSnapshot();
-});
-
-test("run: should fetch the latest event date if no date provided", async () => {
-  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
-  mock_pgQuery.mockReturnValue({ rows: [] });
-
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, [
-      {
-        ...matomoVisit,
-        idVisit: 123,
-      },
-      {
-        ...matomoVisit,
-        idVisit: 124,
-      },
-    ]);
-  });
-
-  await run();
-
-  // check matomo requests
-  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
-  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
-
-  // check db queries
-  expect(mock_pgQuery.mock.calls[NB_REQUEST_TO_INIT_DB][0]).toEqual(
-    "select action_timestamp from matomo order by action_timestamp desc limit 1"
-  );
-});
-
-test("run: should resume using latest event date - offset if no date provided", async () => {
-  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
-
-  const LAST_EVENT_DATE_OFFSET = 2;
-  // @ts-ignore
-  const LAST_EVENT_DATE = addDays(TEST_DATE, -LAST_EVENT_DATE_OFFSET);
-
-  mock_pgQuery.mockReturnValue({ rows: [{ action_timestamp: LAST_EVENT_DATE.getTime() }] });
-
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, [
-      {
-        ...matomoVisit,
-        idVisit: 123,
-      },
-      {
-        ...matomoVisit,
-        idVisit: 124,
-      },
-    ]);
-  });
-
-  await run();
-
-  // check matomo requests
-  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(
-    // @ts-ignore
-    isoDate(addDays(LAST_EVENT_DATE, -parseInt(OFFSET)))
-  );
-  expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
-
-  const daysCount = LAST_EVENT_DATE_OFFSET + parseInt(OFFSET) + 1;
-  expect(mock_matomoApi.mock.calls.length).toEqual(daysCount);
-
-  // check db queries
-  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + daysCount * 7); // NB_REQUEST_TO_INIT_DB + select queries + days offset
-});
-
-test("run: should use today date if nothing in DB", async () => {
-  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
-  mock_pgQuery.mockReturnValue({ rows: [] });
-
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, [
-      {
-        ...matomoVisit,
-        idVisit: 123,
-      },
-    ]);
-  });
-
-  await run();
-
-  // check matomo requests
-  expect(mock_matomoApi.mock.calls.length).toEqual(1);
-  expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
-
-  // check the 4 events inserted
-  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 5); // NB_REQUEST_TO_INIT_DB + check date + latest + 2 inserts
-});
-
-test("run: should use given date if any", async () => {
-  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
-  mock_pgQuery.mockReturnValue({ rows: [] });
-
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, [
-      {
-        ...matomoVisit,
-        idVisit: 123,
-      },
-    ]);
-  });
-
-  // @ts-ignore
-  await run(isoDate(addDays(TEST_DATE, -10)) + "T00:00:00.000Z");
-
-  expect(mock_matomoApi.mock.calls.length).toEqual(11);
-  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 11 * 4); // NB_REQUEST_TO_INIT_DB + inserts. no initial select as date is provided
-});
-
-test("run: should use STARTDATE if any", async () => {
-  // @ts-ignore
-  process.env.STARTDATE = isoDate(addDays(TEST_DATE, -5)) + "T00:00:00.000Z";
-  jest.useFakeTimers("modern").setSystemTime(TEST_DATE.getTime());
-  mock_pgQuery.mockReturnValue({ rows: [] });
-
-  mock_matomoApi.mockImplementation((options, cb) => {
-    return cb(null, [
-      {
-        ...matomoVisit,
-        idVisit: 123,
-      },
-    ]);
-  });
-
-  await run();
-
-  expect(mock_matomoApi.mock.calls.length).toEqual(6);
-
-  expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + 6 * 4); // NB_REQUEST_TO_INIT_DB + initial select + inserts.
-});

package/src/__tests__/visit.json

@@ -1,103 +0,0 @@
-{
-  "idVisit": "124",
-  "idSite": "42",
-  "country": "Argentine",
-  "operatingSystemName": "Mac",
-  "deviceBrand": "Inconnu",
-  "deviceModel": "Générique Bureau",
-  "visitDuration": "300",
-  "daysSinceFirstVisit": "23",
-  "actions": "2",
-  "visitorType": "returningCustomer",
-  "visitorId": "visitorId",
-  "referrerType": "referrerType",
-  "referrerName": "referrerName",
-  "siteName": "tests",
-  "userId": "24",
-  "region": "Buenos Aires",
-  "city": "Buenos Aires",
-  "dimension1": "guest",
-  "dimension3": "page",
-  "dimension6": "shop",
-  "dimension7": "v1.2.3",
-  "dimension8": "fr",
-  "dimension9": "light",
-  "dimension10": "36",
-  "firstActionTimestamp": 1629496512,
-  "actionDetails": [
-    {
-      "type": "event",
-      "url": "https://dive-shop.net/products/basic-wetsuit/",
-      "pageIdAction": "304",
-      "idpageview": "",
-      "serverTimePretty": "20 août 2021 21:35:18",
-      "pageId": "19696671",
-      "eventCategory": "Ecommerce",
-      "eventAction": "Cart change",
-      "timeSpent": 48,
-      "timeSpentPretty": "48s",
-      "pageviewPosition": "8",
-      "timestamp": 1629495318,
-      "icon": "plugins/Morpheus/images/event.png",
-      "iconSVG": "plugins/Morpheus/images/event.svg",
-      "title": "Evènement",
-      "subtitle": "Catégorie: \"Ecommerce', Action: \"Cart change\"",
-      "eventName": "added - Basic Wetsuit",
-      "eventValue": 1,
-      "dimension2": "julien",
-      "dimension4": "indonesia",
-      "dimension5": "diving",
-      "customVariables": {
-        "1": {
-          "customVariableName1": "page-author",
-          "customVariableValue1": "Julien"
-        },
-        "2": {
-          "customVariableName2": "post-age",
-          "customVariableValue2": "-430 days"
-        }
-      }
-    },
-    {
-      "type": "action",
-      "url": "https://dive-shop.net/products/diving-boots/",
-      "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
-      "pageIdAction": "60",
-      "idpageview": "8CDIez",
-      "serverTimePretty": "20 août 2021 21:30:25",
-      "pageId": "19696664",
-      "timeSpent": "2",
-      "timeSpentPretty": "2s",
-      "pageviewPosition": "5",
-      "title": "Divezone Brand Diving Boots - Divezone Store",
-      "subtitle": "https://dive-shop.net/products/diving-boots/",
-      "icon": "",
-      "iconSVG": "plugins/Morpheus/images/action.svg",
-      "timestamp": 1629495025,
-      "dimension2": "julien",
-      "dimension4": "indonesia",
-      "dimension5": "diving"
-    },
-    {
-      "type": "search",
-      "url": "https://dive-shop.net/products/diving-boots/",
-      "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
-      "pageIdAction": "60",
-      "idpageview": "8CDIez",
-      "serverTimePretty": "20 août 2021 21:30:25",
-      "pageId": "19696664",
-      "timeSpent": "2",
-      "timeSpentPretty": "2s",
-      "pageviewPosition": "5",
-      "title": "Divezone Brand Diving Boots - Divezone Store",
-      "subtitle": "https://dive-shop.net/products/diving-boots/",
-      "icon": "",
-      "iconSVG": "plugins/Morpheus/images/action.svg",
-      "timestamp": 1629495022,
-      "dimension2": "julien",
-      "dimension4": "indonesia",
-      "dimension5": "diving",
-      "siteSearchKeyword": "scuba"
-    }
-  ]
-}

package/src/config.js

@@ -1,9 +0,0 @@
-const MATOMO_KEY = process.env.MATOMO_KEY || "";
-const MATOMO_URL = process.env.MATOMO_URL || "https://matomo.fabrique.social.gouv.fr/";
-const MATOMO_SITE = process.env.MATOMO_SITE || 0;
-const PGDATABASE = process.env.PGDATABASE || "";
-const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
-const OFFSET = process.env.OFFSET || "3";
-const RESULTPERPAGE = process.env.RESULTPERPAGE || "500";
-
-module.exports = { MATOMO_KEY, MATOMO_URL, MATOMO_SITE, PGDATABASE, DESTINATION_TABLE, OFFSET, RESULTPERPAGE };

package/src/createTable.js

@@ -1,83 +0,0 @@
-const { Client } = require("pg");
-
-const { DESTINATION_TABLE } = require("./config");
-
-/**
- *
- * @param {Client} client
- */
-async function createTable(client) {
-  const table = client.escapeIdentifier(DESTINATION_TABLE);
-  const text = `
-
-CREATE SCHEMA IF NOT EXISTS partman;
-CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman;
-CREATE TABLE IF NOT EXISTS ${table}
-(
-    idsite                      text,
-    idvisit                     text,
-    actions                     text,
-    country                     text,
-    region                      text,
-    city                        text,
-    operatingsystemname         text,
-    devicemodel                 text,
-    devicebrand                 text,
-    visitduration               text,
-    dayssincefirstvisit         text,
-    visitortype                 text,
-    sitename                    text,
-    userid                      text,
-    serverdateprettyfirstaction date,
-    action_id                   text,
-    action_type                 text,
-    action_eventcategory        text,
-    action_eventaction          text,
-    action_eventname            text,
-    action_eventvalue           decimal,
-    action_timespent            text,
-    action_timestamp            timestamp with time zone DEFAULT now(),
-    usercustomproperties        json,
-    usercustomdimensions        json,
-    dimension1                  text,
-    dimension2                  text,
-    dimension3                  text,
-    dimension4                  text,
-    dimension5                  text,
-    dimension6                  text,
-    dimension7                  text,
-    dimension8                  text,
-    dimension9                  text,
-    dimension10                 text,
-    action_url                  text,
-    sitesearchkeyword           text,
-    action_title                text,
-    visitorid                   text,
-    referrertype                text,
-    referrername                text
-) PARTITION BY RANGE (action_timestamp);
-`;
-
-  await client.query(text, []);
-
-  const migrations = [
-    `CREATE INDEX IF NOT EXISTS idx_dimension1 ON matomo(dimension1);
-     CREATE INDEX IF NOT EXISTS idx_dimension2 ON matomo(dimension2);
-     CREATE INDEX IF NOT EXISTS idx_dimension3 ON matomo(dimension3);
-     CREATE INDEX IF NOT EXISTS idx_dimension4 ON matomo(dimension4);
-     CREATE INDEX IF NOT EXISTS idx_dimension5 ON matomo(dimension5);
-     CREATE INDEX IF NOT EXISTS idx_userid ON matomo(userid);
-     CREATE INDEX IF NOT EXISTS idx_actionurl ON matomo(action_url);
-     CREATE INDEX IF NOT EXISTS idx_region ON matomo(region);`,
-    `ALTER TABLE matomo ADD COLUMN IF NOT EXISTS visitorid text;
-     ALTER TABLE matomo ADD COLUMN IF NOT EXISTS referrertype text;
-     ALTER TABLE matomo ADD COLUMN IF NOT EXISTS referrername text;
-     CREATE INDEX IF NOT EXISTS idx_visitorid ON matomo(visitorid);`,
-  ];
-
-  for (const query of migrations) {
-    await client.query(query, []);
-  }
-}
-
-module.exports = { createTable };

package/src/importDate.js

@@ -1,105 +0,0 @@
-const { Client } = require("pg");
-const pAll = require("p-all");
-const debug = require("debug")("importDate");
-const formatISO = require("date-fns/formatISO");
-
-const { importEvent, getEventsFromMatomoVisit } = require("./importEvent");
-
-const { RESULTPERPAGE, MATOMO_SITE, DESTINATION_TABLE } = require("./config");
-
-/**
- * return date as ISO yyyy-mm-dd
- *
- * @param {Date} date date
- *
- * @returns {string}
- */
-// @ts-ignore
-const isoDate = (date) => formatISO(date, { representation: "date" });
-
-/**
- * check count if imported rows for given date
- *
- * @param {Client} client PG client
- * @param {Date} date datetime as ISO string
- *
- * @returns {Promise<any>}
- */
-const getRecordsCount = async (client, date) => {
-  const text = `SELECT COUNT(distinct idvisit) FROM ${client.escapeIdentifier(
-    DESTINATION_TABLE
-  )} WHERE action_timestamp::date='${isoDate(date)}';`;
-  const result = await client.query(text);
-  return parseInt((result && result.rows && result.rows.length && result.rows[0].count) || 0);
-};
-
-/**
- * import all matomo data for a given date
- *
- * @param {Client} client PG client
- * @param {any} piwikApi piwik.api instance
- * @param {Date} date datetime as ISO string
- *
- * @returns {Promise<any[]>}
- */
-const importDate = async (client, piwikApi, date, filterOffset = 0) => {
-  const limit = parseInt(RESULTPERPAGE);
-  const offset = filterOffset || (await getRecordsCount(client, date));
-  if (!offset) {
-    debug(`${isoDate(date)}: load ${limit} visits`);
-  } else {
-    debug(`${isoDate(date)}: load ${limit} more visits after ${offset}`);
-  }
-  // fetch visits details
-  const visits = await new Promise((resolve) =>
-    piwikApi(
-      {
-        method: "Live.getLastVisitsDetails",
-        period: "day",
-        date: isoDate(date),
-        filter_limit: limit,
-        filter_offset: offset,
-        filter_sort_order: "asc",
-        idSite: MATOMO_SITE,
-      },
-      (err, responseObject = []) => {
-        if (err) {
-          console.error("err", err);
-          resolve([]);
-        }
-        return resolve(responseObject || []);
-      }
-    )
-  );
-
-  debug(`fetched ${visits.length} visits`);
-
-  // flatten events
-  const allEvents = visits.flatMap(getEventsFromMatomoVisit);
-
-  if (!allEvents.length) {
-    debug(`no more valid events after ${isoDate(date)}`);
-    return [];
-  }
-
-  debug(`import ${allEvents.length} events`);
-
-  // serial-import events into PG
-  const importedEvents = await pAll(
-    allEvents.map((event) => () => importEvent(client, event)),
-    { concurrency: 10, stopOnError: true }
-  );
-
-  // continue to next page if necessary
-  if (visits.length === limit) {
-    const nextOffset = offset + limit;
-    const nextEvents = await importDate(client, piwikApi, date, nextOffset);
-    return [...importedEvents, ...(nextEvents || [])];
-  }
-
-  debug(`finished importing ${isoDate(date)}, offset ${offset}`);
-
-  return importedEvents || [];
-};
-
-module.exports = { importDate };

package/src/importEvent.js

@@ -1,108 +0,0 @@
-const { Client } = require("pg");
-
-const { DESTINATION_TABLE } = require("./config");
-
-/**
- *
- * @param {Client} client
- * @param {import("types").Event} event
- *
- * @return {Promise<Record<"rows", any[]>>}
- */
-const importEvent = (client, event) => {
-  const eventKeys = Object.keys(event);
-  const text = `insert into ${client.escapeIdentifier(DESTINATION_TABLE)}
-        (${eventKeys.join(", ")})
-        values (${eventKeys.map((k, i) => `\$${i + 1}`).join(", ")})
-        ON CONFLICT DO NOTHING`;
-  const values = [...eventKeys.map((k) => event[k])];
-  return client.query(text, values).catch((e) => {
-    console.log("QUERY error", e);
-    return { rows: [] };
-  });
-};
-
-const matomoProps = [
-  "idSite",
-  "idVisit",
-  "actions",
-  "country",
-  "region",
-  "city",
-  "operatingSystemName",
-  "deviceModel",
-  "deviceBrand",
-  "visitDuration",
-  "daysSinceFirstVisit",
-  "visitorType",
-  "visitorId",
-  "referrerType",
-  "referrerName",
-  "siteName",
-  "userId",
-];
-
-/** @type Record<string, (a: import("types/matomo").ActionDetail) => string | number> */
-const actionProps = {
-  action_type: (action) => action.type,
-  action_title: (action) => action.title,
-  action_eventcategory: (action) => action.eventCategory,
-  action_eventaction: (action) => action.eventAction,
-  action_eventname: (action) => action.eventName,
-  action_eventvalue: (action) => action.eventValue,
-  action_timespent: (action) => action.timeSpent,
-  action_timestamp: (action) => new Date(action.timestamp * 1000).toISOString(),
-  action_url: (action) => action.url,
-  siteSearchKeyword: (action) => action.siteSearchKeyword,
-};
-
-/**
- * Convert a single matomo visit {<import("types/matomo").Visit} to multiple {Event[]}
- *
- * @param {Partial<import("types/matomo").Visit>} matomoVisit
- *
- * @return {import("types").Event[]} list of events
- *
- */
-const getEventsFromMatomoVisit = (matomoVisit) => {
-  return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
-    /** @type {Record<string, string>} */
-    const usercustomproperties = {};
-    for (let k = 1; k < 10; k++) {
-      const property = actionDetail.customVariables && actionDetail.customVariables[k];
-      if (!property) continue; // max 10 custom variables
-      usercustomproperties[property[`customVariableName${k}`]] = property[`customVariableValue${k}`];
-    }
-
-    /** @type {Record<string, string>} */
-    const usercustomdimensions = {};
-    for (let k = 1; k < 11; k++) {
-      const dimension = `dimension${k}`;
-      const value = actionDetail[dimension] || matomoVisit[dimension];
-      if (!value) continue; // max 10 custom variables
-      usercustomdimensions[dimension] = value;
-    }
-
-    /** @type {import("types").Event} */
-    // @ts-ignore
-    const event = {
-      // default matomo visit properties
-      ...matomoProps.reduce((a, prop) => ({ ...a, [prop.toLowerCase()]: matomoVisit[prop] }), {}),
-      serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString(),
-      // action specific properties
-      ...Object.keys(actionProps).reduce((a, prop) => ({ ...a, [prop]: actionProps[prop](actionDetail) }), {
-        action_id: `${matomoVisit.idVisit}_${actionIndex}`,
-      }),
-      // custom variables
-      usercustomproperties,
-      // custom dimensions
-      // We keep both for backwards compatibility.
-      // Current implementation is flat with one column for each dimension.
-      usercustomdimensions,
-      ...usercustomdimensions,
-    };
-    return event;
-  });
-};
-
-module.exports = { getEventsFromMatomoVisit, importEvent };

package/src/index.js

@@ -1,79 +0,0 @@
-const pAll = require("p-all");
-const debug = require("debug")("index");
-const eachDayOfInterval = require("date-fns/eachDayOfInterval");
-const PiwikClient = require("piwik-client");
-const { Client } = require("pg");
-
-const { MATOMO_KEY, MATOMO_URL, MATOMO_SITE, PGDATABASE, DESTINATION_TABLE, OFFSET } = require("./config");
-
-const { createTable } = require("./createTable");
-const { importDate } = require("./importDate");
-
-// run a sync with a 3-days range
-/**
- *
- * @param {string} [date]
- * @returns
- */
-async function run(date) {
-  debug("run, date=" + date);
-  const client = new Client({ connectionString: PGDATABASE });
-  await client.connect();
-
-  const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
-
-  await createTable(client);
-
-  // priority:
-  //  - optional parameter date
-  //  - last event in the table
-  //  - optional env.STARTDATE
-  //  - today
-
-  let referenceDate;
-  if (!referenceDate && date) referenceDate = new Date(date);
-  if (!referenceDate) referenceDate = await findLastEventInMatomo(client);
-  if (!referenceDate && process.env.STARTDATE) referenceDate = new Date(process.env.STARTDATE);
-  if (!referenceDate) referenceDate = new Date();
-
-  // @ts-ignore
-  const dates = eachDayOfInterval({
-    start: referenceDate,
-    end: new Date(),
-  });
-
-  debug(`import : ${dates.join(", ")}`);
-
-  // for each date, serial-import data
-  const res = await pAll(
-    dates.map((date) => () => importDate(client, piwik.api.bind(piwik), date)),
-    { concurrency: 1, stopOnError: true }
-  );
-
-  await client.end();
-  debug("close");
-
-  return res;
-}
-
-module.exports = run;
-
-if (require.main === module) {
-  (async () => {
-    if (!MATOMO_SITE) return console.error("Missing env MATOMO_SITE");
-    if (!MATOMO_KEY) return console.error("Missing env MATOMO_KEY");
-    if (!PGDATABASE) return console.error("Missing env PGDATABASE");
-    await run();
-    debug("run finished");
-  })();
-}
-
-async function findLastEventInMatomo(client) {
-  const a = await client.query(
-    `select action_timestamp from ${client.escapeIdentifier(DESTINATION_TABLE)} order by action_timestamp desc limit 1`
-  );
-  if (!a.rows.length || !a.rows[0].action_timestamp) return null;
-  const d = new Date(a.rows[0].action_timestamp);
-  d.setDate(d.getDate() - +OFFSET);
-  return d;
-}