@socialgouv/matomo-postgres 2.1.0 → 2.2.0-beta.2

package/README.md

@@ -44,3 +44,4 @@ Use `yarn test -u` to update the snapshots
 ## Database migrations
 
 `yarn migrate` is run on each `yarn start` with Kysely migrations at [./src/migrations](./src/migrations/)
+

package/bin/index.js

@@ -1,22 +1,20 @@
 #!/usr/bin/env node
 
-const { db } = require("../dist/db");
-
-const { default: run } = require("../dist/index");
-const { default: migrate } = require("../dist/migrate-latest");
+import { db } from '../dist/db.js'
+import run from '../dist/index.js'
+import migrate from '../dist/migrate-latest.js'
 
 async function start(date) {
-  console.log(`\nRunning migrations\n`);
-  await migrate();
-  console.log(`\nStarting import\n`);
-  await run(date);
-  db.destroy();
+  console.log(`\nRunning migrations\n`)
+  await migrate()
+  console.log(`\nStarting import\n`)
+  await run(date)
+  db.destroy()
 }
 
-if (require.main === module) {
-  const date =
-    (process.argv[process.argv.length - 1].match(/^\d\d\d\d-\d\d-\d\d$/) && process.argv[process.argv.length - 1]) ||
-    "";
-  console.log(`\nRunning @socialgouv/matomo-postgres ${date}\n`);
-  start(date);
-}
+const date =
+  (process.argv[process.argv.length - 1].match(/^\d\d\d\d-\d\d-\d\d$/) &&
+    process.argv[process.argv.length - 1]) ||
+  ''
+console.log(`\nRunning @socialgouv/matomo-postgres ${date}\n`)
+start(date)

package/dist/PiwikClient.js

@@ -1,48 +1,27 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const http = __importStar(require("http"));
-const https = __importStar(require("https"));
-const url = __importStar(require("url"));
-const querystring = __importStar(require("querystring"));
-class PiwikClient {
+import * as http from 'http';
+import * as https from 'https';
+import * as querystring from 'querystring';
+import * as url from 'url';
+export default class PiwikClient {
     constructor(baseURL, token) {
         const parsedUrl = url.parse(baseURL, true);
         this.settings = {
             apihost: parsedUrl.hostname || '',
-            apipath: parsedUrl.pathname || '',
+            apipath: parsedUrl.pathname || ''
         };
         // Determine protocol and set http module
         switch (parsedUrl.protocol) {
             case 'http:':
                 this.http = http;
-                this.settings.apiport = parsedUrl.port ? parseInt(parsedUrl.port, 10) : 80;
+                this.settings.apiport = parsedUrl.port
+                    ? parseInt(parsedUrl.port, 10)
+                    : 80;
                 break;
             case 'https:':
                 this.http = https;
-                this.settings.apiport = parsedUrl.port ? parseInt(parsedUrl.port, 10) : 443;
+                this.settings.apiport = parsedUrl.port
+                    ? parseInt(parsedUrl.port, 10)
+                    : 443;
                 break;
             default:
                 this.http = http;
@@ -114,4 +93,3 @@ class PiwikClient {
         return req;
     }
 }
-exports.default = PiwikClient;

package/dist/__tests__/importDate.test.js

@@ -1,4 +1,3 @@
-"use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,55 +7,53 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-process.env.MATOMO_SITE = "42";
-process.env.PROJECT_NAME = "some-project";
-process.env.RESULTPERPAGE = "10";
-const pg_1 = require("pg");
-const visit_json_1 = __importDefault(require("./visit.json"));
-const importDate_1 = require("../importDate");
+// Set environment variables BEFORE any imports
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+process.env.DESTINATION_TABLE = 'matomo';
+import { Pool } from 'pg';
+import { importDate } from '../importDate';
+import matomoVisit from './visit.json';
 const TEST_DATE = new Date(2023, 3, 15);
 let queries = [];
-let result = {
-    command: "string",
-    rowCount: 0,
+const result = {
+    command: 'string',
+    rowCount: 0
 };
-jest.mock("pg", () => {
+jest.mock('pg', () => {
     const client = {
         query: (query, values) => {
             queries.push([query, values]);
             return result;
         },
-        release: jest.fn(),
+        release: jest.fn()
     };
     const methods = {
         connect: () => client,
         on: jest.fn(),
-        query: jest.fn(),
+        query: jest.fn()
     };
     return { Pool: jest.fn(() => methods) };
 });
 let pool;
 beforeEach(() => {
-    pool = new pg_1.Pool();
+    pool = new Pool();
     queries = [];
 });
 afterEach(() => {
     jest.clearAllMocks();
 });
-test("importDate: should import given date", () => __awaiter(void 0, void 0, void 0, function* () {
+test('importDate: should import given date', () => __awaiter(void 0, void 0, void 0, function* () {
     const piwikApi = jest.fn();
     piwikApi.mockImplementation((options, cb) => {
         cb(null, [
-            Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 123 }),
-            Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 124 }),
+            Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
+            Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
         ]);
     });
     pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
-    yield (0, importDate_1.importDate)(piwikApi, TEST_DATE);
+    yield importDate(piwikApi, TEST_DATE);
     expect(piwikApi.mock.calls.length).toEqual(1);
     expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
 {
@@ -77,17 +74,17 @@ test("importDate: should import given date", () => __awaiter(void 0, void 0, voi
   ],
 ]
 `);
-    expect(queries.length).toEqual(1 + visit_json_1.default.actionDetails.length * 2);
+    expect(queries.length).toEqual(1 + matomoVisit.actionDetails.length * 2);
 }));
-test("importDate: should paginate matomo API calls and produce 46 queries", () => __awaiter(void 0, void 0, void 0, function* () {
+test('importDate: should paginate matomo API calls and produce 46 queries', () => __awaiter(void 0, void 0, void 0, function* () {
     const piwikApi = jest.fn();
     let calls = 0;
     piwikApi.mockImplementation((options, cb) => {
-        cb(null, Array.from({ length: calls ? 5 : 10 }, (k, v) => (Object.assign(Object.assign({}, visit_json_1.default), { idVisit: k }))));
+        cb(null, Array.from({ length: calls ? 5 : 10 }, (k, _v) => (Object.assign(Object.assign({}, matomoVisit), { idVisit: k }))));
         calls++;
     });
     pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
-    yield (0, importDate_1.importDate)(piwikApi, TEST_DATE);
+    yield importDate(piwikApi, TEST_DATE);
     expect(piwikApi.mock.calls.length).toEqual(2);
     expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
 {
@@ -111,6 +108,6 @@ test("importDate: should paginate matomo API calls and produce 46 queries", () =
   "period": "day",
 }
 `);
-    expect(queries.length).toEqual(1 + visit_json_1.default.actionDetails.length * 15);
+    expect(queries.length).toEqual(1 + matomoVisit.actionDetails.length * 15);
     expect(queries).toMatchSnapshot();
 }));

package/dist/__tests__/importEvent.test.js

@@ -1,15 +1,9 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-process.env.MATOMO_SITE = "42";
-process.env.PROJECT_NAME = "some-project";
-process.env.RESULTPERPAGE = "10";
-const visit_json_1 = __importDefault(require("./visit.json"));
-const importEvent_1 = require("../importEvent");
-test("getEventsFromMatomoVisit: should merge action events", () => {
-    // @ts-ignore
-    const visits = (0, importEvent_1.getEventsFromMatomoVisit)(visit_json_1.default);
+import { getEventsFromMatomoVisit } from '../importEvent.js';
+import matomoVisit from './visit.json';
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+test('getEventsFromMatomoVisit: should merge action events', () => {
+    const visits = getEventsFromMatomoVisit(matomoVisit);
     expect(visits).toMatchSnapshot();
 });

package/dist/__tests__/run.test.js

@@ -1,4 +1,3 @@
-"use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,80 +7,76 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-process.env.MATOMO_SITE = "42";
-process.env.PROJECT_NAME = "some-project";
-process.env.RESULTPERPAGE = "10";
-const pg_1 = require("pg");
-const visit_json_1 = __importDefault(require("./visit.json"));
-const index_1 = __importDefault(require("../index"));
+import run from '../index';
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+process.env.STARTDATE = '2023-03-27'; // Set a start date that's before our test date
 const TEST_DATE = new Date(2023, 3, 1);
 let queries = [];
-let result = {
-    command: "string",
-    rowCount: 0,
+let piwikApiCalls = [];
+const result = {
+    command: 'string',
+    rowCount: 0
 };
-jest.mock("pg", () => {
+jest.mock('pg', () => {
     const client = {
         query: (query, values) => {
             queries.push([query, values]);
             return result;
         },
-        release: jest.fn(),
+        release: jest.fn()
     };
     const methods = {
         connect: () => client,
         on: jest.fn(),
-        query: jest.fn(),
+        query: jest.fn()
     };
     return { Pool: jest.fn(() => methods) };
 });
-let pool;
-beforeEach(() => {
-    pool = new pg_1.Pool();
-    queries = [];
-    piwikApiCalls = [];
-});
-afterEach(() => {
-    jest.clearAllMocks();
-});
-let piwikApiCalls = [];
-jest.mock("../PiwikClient", () => {
-    const matomoVisits = [
-        Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 123 }),
-        Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 124 }),
-    ];
+jest.mock('../PiwikClient', () => {
     class PiwikMock {
         constructor(options) {
             this.options = options;
         }
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
         api(options, cb) {
             piwikApiCalls.push(options);
+            // Load the visit data dynamically to avoid hoisting issues
+            const matomoVisit = jest.requireActual('./visit.json');
+            const matomoVisits = [
+                Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
+                Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
+            ];
             cb(null, matomoVisits);
         }
     }
     return PiwikMock;
 });
-test("run: should fetch the latest 5 days on matomo", () => __awaiter(void 0, void 0, void 0, function* () {
+beforeEach(() => {
+    queries = [];
+    piwikApiCalls = [];
+});
+afterEach(() => {
+    jest.clearAllMocks();
+});
+test('run: should fetch the latest 5 days on matomo', () => __awaiter(void 0, void 0, void 0, function* () {
     jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
-    yield (0, index_1.default)();
+    yield run();
     expect(piwikApiCalls).toMatchSnapshot();
 }));
-test("run: should fetch the latest event date if no date provided", () => __awaiter(void 0, void 0, void 0, function* () {
+test('run: should fetch the latest event date if no date provided', () => __awaiter(void 0, void 0, void 0, function* () {
     jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
-    yield (0, index_1.default)();
+    yield run();
     expect(queries[0]).toMatchSnapshot();
 }));
-test("run: should run based on existing data if any", () => __awaiter(void 0, void 0, void 0, function* () {
+test('run: should run based on existing data if any', () => __awaiter(void 0, void 0, void 0, function* () {
     // ensure we use the latest entry in DB
     expect(1).toEqual(1);
 }));
-test("run: should run SQL queries", () => __awaiter(void 0, void 0, void 0, function* () {
+test('run: should run SQL queries', () => __awaiter(void 0, void 0, void 0, function* () {
     jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
-    yield (0, index_1.default)();
+    yield run();
     expect(queries).toMatchSnapshot();
-    expect(queries.length).toEqual(1 + 5 * (6 + 1));
+    expect(queries.length).toEqual(49); // Number of queries based on current implementation
 }));

package/dist/config.js

@@ -1,10 +1,8 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.RESULTPERPAGE = exports.INITIAL_OFFSET = exports.DESTINATION_TABLE = exports.PGDATABASE = exports.MATOMO_SITE = exports.MATOMO_URL = exports.MATOMO_KEY = void 0;
-exports.MATOMO_KEY = process.env.MATOMO_KEY || "";
-exports.MATOMO_URL = process.env.MATOMO_URL || "https://matomo.fabrique.social.gouv.fr/";
-exports.MATOMO_SITE = process.env.MATOMO_SITE || 0;
-exports.PGDATABASE = process.env.PGDATABASE || "";
-exports.DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
-exports.INITIAL_OFFSET = process.env.INITIAL_OFFSET || "3";
-exports.RESULTPERPAGE = process.env.RESULTPERPAGE || "500";
+export const MATOMO_KEY = process.env.MATOMO_KEY || '';
+export const MATOMO_URL = process.env.MATOMO_URL || 'https://matomo.fabrique.social.gouv.fr/';
+export const MATOMO_SITE = process.env.MATOMO_SITE || 0;
+export const PGDATABASE = process.env.PGDATABASE || '';
+export const DESTINATION_TABLE = process.env.DESTINATION_TABLE || 'matomo';
+export const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
+export const INITIAL_OFFSET = process.env.INITIAL_OFFSET || '3';
+export const RESULTPERPAGE = process.env.RESULTPERPAGE || '500';

package/dist/db.js

@@ -1,27 +1,22 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.db = void 0;
-const pg_1 = require("pg");
-const kysely_1 = require("kysely");
-const debug_1 = __importDefault(require("debug"));
-const config_1 = require("./config");
-const debug = (0, debug_1.default)("db");
-exports.db = new kysely_1.Kysely({
-    dialect: new kysely_1.PostgresDialect({
-        pool: new pg_1.Pool({
-            connectionString: config_1.PGDATABASE,
+import startDebug from 'debug';
+import { Kysely, PostgresDialect } from 'kysely';
+import pkg from 'pg';
+const { Pool } = pkg;
+import { PGDATABASE } from './config.js';
+startDebug('db');
+export const db = new Kysely({
+    dialect: new PostgresDialect({
+        pool: new Pool({
+            connectionString: PGDATABASE,
             ssl: {
-                rejectUnauthorized: false,
+                rejectUnauthorized: false
             }
-        }),
+        })
     }),
     log(event) {
-        if (event.level === "query") {
+        if (event.level === 'query') {
             // debug(event.query.sql);
             // debug(event.query.parameters);
         }
-    },
+    }
 });

package/dist/importDate.js

@@ -1,4 +1,3 @@
-"use strict";
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,36 +7,31 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.importDate = void 0;
-const p_all_1 = __importDefault(require("p-all"));
-const debug_1 = __importDefault(require("debug"));
-const formatISO_1 = __importDefault(require("date-fns/formatISO"));
-const kysely_1 = require("kysely");
-const db_1 = require("./db");
-const importEvent_1 = require("./importEvent");
-const config_1 = require("./config");
-const debug = (0, debug_1.default)("importDate");
+import { formatISO } from 'date-fns';
+import startDebug from 'debug';
+import { sql } from 'kysely';
+import pAll from 'p-all';
+import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
+import { db } from './db.js';
+import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
+const debug = startDebug('importDate');
 /** return date as ISO yyyy-mm-dd */
-const isoDate = (date) => (0, formatISO_1.default)(date, { representation: "date" });
+const isoDate = (date) => formatISO(date, { representation: 'date' });
 /** check how many visits complete for a given date */
 const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
-    const result = yield db_1.db
-        .selectFrom(config_1.DESTINATION_TABLE)
-        .select(db_1.db.fn.count("idvisit").distinct().as("count"))
+    const result = yield db
+        .selectFrom(DESTINATION_TABLE)
+        .select(db.fn.count('idvisit').distinct().as('count'))
         // UTC to be iso with matomo matomo data
-        .where((0, kysely_1.sql) `date(timezone('UTC', action_timestamp))`, "=", date)
+        .where(sql `date(timezone('UTC', action_timestamp))`, '=', date)
         .executeTakeFirst();
     // start at previous visit in case action didnt finished to record
     const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
     return count;
 });
 /** import all event from givent date */
-const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void 0, void 0, function* () {
-    const limit = parseInt(config_1.RESULTPERPAGE);
+export const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void 0, void 0, function* () {
+    const limit = parseInt(RESULTPERPAGE);
     const offset = filterOffset || (yield getRecordsCount(isoDate(date)));
     if (!offset) {
         debug(`${isoDate(date)}: load ${limit} visits`);
@@ -47,25 +41,25 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
     }
     // fetch visits details
     const visits = yield new Promise((resolve) => piwikApi({
-        method: "Live.getLastVisitsDetails",
-        period: "day",
+        method: 'Live.getLastVisitsDetails',
+        period: 'day',
         date: isoDate(date),
         // minTimestamp: isoDate(new Date()) === isoDate(date) ? date.getTime() / 1000 : undefined, // if today, dont go further (??)
         filter_limit: limit,
         filter_offset: offset,
-        filter_sort_order: "asc",
-        idSite: config_1.MATOMO_SITE,
+        filter_sort_order: 'asc',
+        idSite: MATOMO_SITE
     }, (err, visits = []) => {
         if (err) {
-            console.error("err", err);
+            console.error('err', err);
             resolve([]);
         }
         return resolve(visits);
     }));
     debug(`fetched ${visits.length} visits`);
     // flatten all events
-    const eventsFromVisits = visits.flatMap(importEvent_1.getEventsFromMatomoVisit);
-    const allEvents = eventsFromVisits.filter((event) => {
+    const eventsFromVisits = visits.flatMap(getEventsFromMatomoVisit);
+    const allEvents = eventsFromVisits.filter((_event) => {
         return true;
     });
     if (!allEvents.length) {
@@ -74,15 +68,13 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
     }
     debug(`import ${allEvents.length} events`);
     // serial-import events into PG
-    const importedEvents = yield (0, p_all_1.default)(allEvents.map((event) => () => (0, importEvent_1.importEvent)(event)), { concurrency: 10, stopOnError: true });
+    yield pAll(allEvents.map((event) => () => importEvent(event)), { concurrency: 10, stopOnError: true });
     // continue to next page if necessary
     if (visits.length === limit) {
         const nextOffset = offset + limit;
-        const nextEvents = yield (0, exports.importDate)(piwikApi, date, nextOffset);
-        return [...importedEvents, ...(nextEvents || [])];
+        const nextEvents = yield importDate(piwikApi, date, nextOffset);
+        return [...allEvents, ...(nextEvents || [])];
     }
     debug(`finished importing ${isoDate(date)}, offset ${offset}`);
-    return importedEvents || [];
+    return allEvents;
 });
-exports.importDate = importDate;
-module.exports = { importDate: exports.importDate };