@socialgouv/matomo-postgres 1.1.4

package/src/index.js

@@ -0,0 +1,79 @@
+const pAll = require("p-all");
+const debug = require("debug")("index");
+const eachDayOfInterval = require("date-fns/eachDayOfInterval");
+const PiwikClient = require("piwik-client");
+const { Client } = require("pg");
+
+const { MATOMO_KEY, MATOMO_URL, MATOMO_SITE, PGDATABASE, DESTINATION_TABLE, OFFSET } = require("./config");
+
+const { createTable } = require("./createTable");
+const { importDate } = require("./importDate");
+
+// run a sync with a 3-days range
+/**
+ *
+ * @param {string} [date]
+ * @returns
+ */
+async function run(date) {
+  debug("run, date=" + date);
+  const client = new Client({ connectionString: PGDATABASE });
+  await client.connect();
+
+  const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
+
+  await createTable(client);
+
+  // priority:
+  //  - optional parameter date
+  //  - last event in the table
+  //  - optional env.STARTDATE
+  //  - today
+
+  let referenceDate;
+  if (!referenceDate && date) referenceDate = new Date(date);
+  if (!referenceDate) referenceDate = await findLastEventInMatomo(client);
+  if (!referenceDate && process.env.STARTDATE) referenceDate = new Date(process.env.STARTDATE);
+  if (!referenceDate) referenceDate = new Date();
+
+  // @ts-ignore
+  const dates = eachDayOfInterval({
+    start: referenceDate,
+    end: new Date(),
+  });
+
+  debug(`import : ${dates.join(", ")}`);
+
+  // for each date, serial-import data
+  const res = await pAll(
+    dates.map((date) => () => importDate(client, piwik.api.bind(piwik), date)),
+    { concurrency: 1, stopOnError: true }
+  );
+
+  await client.end();
+  debug("close");
+
+  return res;
+}
+
+module.exports = run;
+
+if (require.main === module) {
+  (async () => {
+    if (!MATOMO_SITE) return console.error("Missing env MATOMO_SITE");
+    if (!MATOMO_KEY) return console.error("Missing env MATOMO_KEY");
+    if (!PGDATABASE) return console.error("Missing env PGDATABASE");
+    await run();
+    debug("run finished");
+  })();
+}
+
+async function findLastEventInMatomo(client) {
+  const a = await client.query(
+    `select action_timestamp from ${client.escapeIdentifier(DESTINATION_TABLE)} order by action_timestamp desc limit 1`
+  );
+  if (!a.rows.length || !a.rows[0].action_timestamp) return null;
+  const d = new Date(a.rows[0].action_timestamp);
+  d.setDate(d.getDate() - +OFFSET);
+  return d;
+}

package/types/index.d.ts

@@ -0,0 +1,30 @@
+export * from "./matomo";
+
+export type Event = {
+  idsite: string;
+  idvisit: string;
+  actions: string;
+  country: string;
+  region: string;
+  city: string;
+  operatingsystemname: string;
+  devicemodel: string;
+  devicebrand: string;
+  visitduration: string;
+  dayssincefirstvisit: string;
+  visitortype: string;
+  sitename: string;
+  userid: string;
+  serverdateprettyfirstaction: string;
+  action_id: string;
+  action_type: string;
+  action_eventcategory: string;
+  action_eventaction: string;
+  action_eventname: string;
+  action_eventvalue: string;
+  action_timespent: string;
+  action_timestamp: string;
+  action_url: string;
+  usercustomproperties: Record<string, string>;
+  usercustomdimensions: Record<string, string>;
+};

package/types/matomo.d.ts

@@ -0,0 +1,208 @@
+export type Visits = Visit[];
+
+export type Visit = {
+  idSite: string;
+  idVisit: string;
+  visitIp: any;
+  visitorId: boolean;
+  fingerprint: boolean;
+  actionDetails: ActionDetail[];
+  goalConversions: number;
+  siteCurrency: string;
+  siteCurrencySymbol: string;
+  serverDate: string;
+  visitServerHour: string;
+  lastActionTimestamp: number;
+  lastActionDateTime: string;
+  siteName: string;
+  serverTimestamp: number;
+  firstActionTimestamp: number;
+  serverTimePretty: string;
+  serverDatePretty: string;
+  serverDatePrettyFirstAction: string;
+  serverTimePrettyFirstAction: string;
+  userId: any;
+  visitorType: string;
+  visitorTypeIcon: string;
+  visitConverted: string;
+  visitConvertedIcon?: string;
+  visitCount: string;
+  visitEcommerceStatus: string;
+  visitEcommerceStatusIcon?: string;
+  daysSinceFirstVisit: number;
+  secondsSinceFirstVisit: string;
+  daysSinceLastEcommerceOrder: number;
+  secondsSinceLastEcommerceOrder: string;
+  visitDuration: string;
+  visitDurationPretty: string;
+  searches: string;
+  actions: string;
+  interactions: string;
+  referrerType: string;
+  referrerTypeName: string;
+  referrerName: string;
+  referrerKeyword: string;
+  referrerKeywordPosition: any;
+  referrerUrl?: string;
+  referrerSearchEngineUrl?: string;
+  referrerSearchEngineIcon?: string;
+  referrerSocialNetworkUrl?: string;
+  referrerSocialNetworkIcon?: string;
+  languageCode: string;
+  language: string;
+  deviceType: string;
+  deviceTypeIcon: string;
+  deviceBrand: string;
+  deviceModel: string;
+  operatingSystem: string;
+  operatingSystemName: string;
+  operatingSystemIcon: string;
+  operatingSystemCode: string;
+  operatingSystemVersion: string;
+  browserFamily: string;
+  browserFamilyDescription: string;
+  browser: string;
+  browserName: string;
+  browserIcon: string;
+  browserCode: string;
+  browserVersion: string;
+  totalEcommerceRevenue: string;
+  totalEcommerceConversions: string;
+  totalEcommerceItems: string;
+  totalAbandonedCartsRevenue: string;
+  totalAbandonedCarts: string;
+  totalAbandonedCartsItems: string;
+  events: string;
+  continent: string;
+  continentCode: string;
+  country: string;
+  countryCode: string;
+  countryFlag: string;
+  region?: string;
+  regionCode?: string;
+  city?: string;
+  location: string;
+  latitude: string;
+  longitude: string;
+  visitLocalTime: string;
+  visitLocalHour: string;
+  daysSinceLastVisit: number;
+  secondsSinceLastVisit: string;
+  resolution: string;
+  plugins: string;
+  pluginsIcons?: PluginsIcon[];
+  dimension1?: string;
+  dimension2: string;
+  dimension3?: string;
+  dimension4?: string;
+  dimension5?: string;
+  experiments: Experiment[];
+  customVariables: any;
+  formConversions: number;
+  sessionReplayUrl: any;
+  campaignId: string;
+  campaignContent: string;
+  campaignKeyword: string;
+  campaignMedium: string;
+  campaignName: string;
+  campaignSource: string;
+  campaignGroup: string;
+  campaignPlacement: string;
+};
+
+export interface ActionDetail {
+  type: string;
+  url?: string;
+  pageTitle?: string;
+  pageIdAction?: string;
+  idpageview?: string;
+  serverTimePretty: string;
+  pageId?: string;
+  pageviewPosition?: string;
+  title: string;
+  subtitle?: string;
+  icon: string;
+  iconSVG?: string;
+  timestamp: number;
+  dimension1?: string;
+  dimension2?: string;
+  dimension3?: string;
+  dimension4?: string;
+  dimension5?: string;
+  formName?: string;
+  formId?: string;
+  formStatus?: string;
+  converted?: string;
+  submitted?: number;
+  timeToFirstSubmission?: string;
+  timeSpent: any;
+  timeHesitation?: string;
+  leftBlank?: number;
+  fields?: Field[];
+  eventCategory?: string;
+  eventAction?: string;
+  timeSpentPretty?: string;
+  eventName?: string;
+  eventValue?: number;
+  revenue: any;
+  items?: string;
+  itemDetails?: ItemDetail[];
+  customVariables?: CustomVariables;
+  server_time?: string;
+  goalName?: string;
+  goalId?: string;
+  goalPageId?: string;
+  orderId?: string;
+  revenueSubTotal?: number;
+  revenueTax?: number;
+  revenueShipping?: number;
+  revenueDiscount?: number;
+}
+
+export interface Field {
+  fieldName: string;
+  timeSpent: string;
+  timeHesitation: string;
+  leftBlank: string;
+  submitted: string;
+}
+
+export interface ItemDetail {
+  itemSKU: string;
+  itemName: string;
+  itemCategory: string;
+  price: any;
+  quantity: string;
+  categories: string[];
+}
+
+export interface CustomVariables {
+  "1": N1;
+  "2": N2;
+}
+
+export interface N1 {
+  customVariablePageName1: string;
+  customVariablePageValue1: string;
+}
+
+export interface N2 {
+  customVariablePageName2: string;
+  customVariablePageValue2: string;
+}
+
+export interface PluginsIcon {
+  pluginIcon: string;
+  pluginName: string;
+}
+
+export interface Experiment {
+  idexperiment: string;
+  name: string;
+  variation: Variation;
+}
+
+export interface Variation {
+  idvariation: any;
+  name: string;
+}