@socialgouv/matomo-postgres 2.3.11 → 2.3.13

package/dist/PiwikClient.js

@@ -0,0 +1,95 @@
+import * as http from 'http';
+import * as https from 'https';
+import * as querystring from 'querystring';
+import * as url from 'url';
+export default class PiwikClient {
+    constructor(baseURL, token) {
+        const parsedUrl = url.parse(baseURL, true);
+        this.settings = {
+            apihost: parsedUrl.hostname || '',
+            apipath: parsedUrl.pathname || ''
+        };
+        // Determine protocol and set http module
+        switch (parsedUrl.protocol) {
+            case 'http:':
+                this.http = http;
+                this.settings.apiport = parsedUrl.port
+                    ? parseInt(parsedUrl.port, 10)
+                    : 80;
+                break;
+            case 'https:':
+                this.http = https;
+                this.settings.apiport = parsedUrl.port
+                    ? parseInt(parsedUrl.port, 10)
+                    : 443;
+                break;
+            default:
+                this.http = http;
+                this.settings.apiport = 80;
+        }
+        // Set token from URL query or constructor parameter
+        if (parsedUrl.query && parsedUrl.query.token_auth) {
+            this.settings.token = parsedUrl.query.token_auth;
+        }
+        if (token) {
+            this.settings.token = token;
+        }
+    }
+    api(vars, cb) {
+        if (typeof vars !== 'object') {
+            vars = {};
+        }
+        // Set default values
+        vars.module = 'API';
+        vars.format = 'JSON';
+        // Set token if not provided in vars
+        if (vars.token_auth == null) {
+            vars.token_auth = this.settings.token;
+        }
+        // Extract token_auth for POST body
+        const token_auth = vars.token_auth;
+        const postData = querystring.stringify({ token_auth });
+        // Remove token_auth from URL query params
+        const queryVars = Object.assign({}, vars);
+        delete queryVars.token_auth;
+        // Prepare request options
+        const options = {
+            host: this.settings.apihost,
+            port: this.settings.apiport,
+            path: this.settings.apipath + '?' + querystring.stringify(queryVars),
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/x-www-form-urlencoded'
+            }
+        };
+        // Make HTTP POST request
+        const req = this.http.request(options, (response) => {
+            let data = '';
+            // Collect data chunks
+            response.on('data', (chunk) => {
+                data += chunk;
+            });
+            // Process complete response
+            response.on('end', () => {
+                try {
+                    const resObj = JSON.parse(data);
+                    if (resObj.result === 'error') {
+                        return cb(new Error(resObj.message), null);
+                    }
+                    return cb(null, resObj);
+                }
+                catch (error) {
+                    return cb(error instanceof Error ? error : new Error(String(error)), null);
+                }
+            });
+        });
+        // Handle request errors
+        req.on('error', (error) => {
+            cb(error, null);
+        });
+        // Write POST data and end request
+        req.write(postData);
+        req.end();
+        return req;
+    }
+}

package/dist/__tests__/importDate.test.js

@@ -0,0 +1,134 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+// Set environment variables BEFORE any imports
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+process.env.DESTINATION_TABLE = 'matomo';
+import { Pool } from 'pg';
+import { importDate } from '../importDate';
+import matomoVisit from './visit.json';
+const TEST_DATE = new Date(2023, 3, 15);
+let queries = [];
+const result = {
+    command: 'string',
+    rowCount: 0
+};
+jest.mock('pg', () => {
+    const client = {
+        query: (query, values) => {
+            queries.push([query, values]);
+            return result;
+        },
+        release: jest.fn()
+    };
+    const methods = {
+        connect: () => client,
+        on: jest.fn(),
+        query: jest.fn()
+    };
+    return { Pool: jest.fn(() => methods) };
+});
+let pool;
+beforeEach(() => {
+    pool = new Pool();
+    queries = [];
+});
+afterEach(() => {
+    jest.clearAllMocks();
+});
+test('importDate: should import given date', () => __awaiter(void 0, void 0, void 0, function* () {
+    const piwikApi = jest.fn();
+    piwikApi.mockImplementation((options, cb) => {
+        cb(null, [
+            Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
+            Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
+        ]);
+    });
+    pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
+    const result = yield importDate(piwikApi, TEST_DATE);
+    expect(piwikApi.mock.calls.length).toEqual(1);
+    expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
+{
+  "date": "2023-04-15",
+  "filter_limit": 10,
+  "filter_offset": 0,
+  "filter_sort_order": "asc",
+  "idSite": "42",
+  "method": "Live.getLastVisitsDetails",
+  "period": "day",
+}
+`);
+    expect(queries[0]).toMatchInlineSnapshot(`
+[
+  "select count(distinct "idvisit") as "count" from "matomo" where date(timezone('UTC', action_timestamp)) = $1",
+  [
+    "2023-04-15",
+  ],
+]
+`);
+    expect(queries.length).toEqual(1 + matomoVisit.actionDetails.length * 2);
+    expect(result).toMatchObject({
+        date: '2023-04-15',
+        pages: 1,
+        visitsFetched: 2,
+        eventsImported: matomoVisit.actionDetails.length * 2
+    });
+}));
+test('importDate: should handle pagination across multiple pages', () => __awaiter(void 0, void 0, void 0, function* () {
+    const piwikApi = jest.fn();
+    // Mock first call to return exactly 10 visits (triggers pagination)
+    piwikApi
+        .mockImplementationOnce((options, cb) => {
+        const visits = Array.from({ length: 10 }, (_, i) => (Object.assign(Object.assign({}, matomoVisit), { idVisit: 200 + i })));
+        cb(null, visits);
+    })
+        // Mock second call to return 5 visits (stops pagination)
+        .mockImplementationOnce((options, cb) => {
+        const visits = Array.from({ length: 5 }, (_, i) => (Object.assign(Object.assign({}, matomoVisit), { idVisit: 300 + i })));
+        cb(null, visits);
+    });
+    // Mock database query for record count
+    pool.query.mockResolvedValue({ rows: [], rowCount: 0 });
+    const result = yield importDate(piwikApi, TEST_DATE);
+    // Should make exactly 2 API calls due to pagination
+    expect(piwikApi.mock.calls.length).toEqual(2);
+    // First call should have offset 0
+    expect(piwikApi.mock.calls[0][0]).toMatchObject({
+        date: '2023-04-15',
+        filter_limit: 10,
+        filter_offset: 0,
+        filter_sort_order: 'asc',
+        idSite: '42',
+        method: 'Live.getLastVisitsDetails',
+        period: 'day'
+    });
+    // Second call should have offset 10
+    expect(piwikApi.mock.calls[1][0]).toMatchObject({
+        date: '2023-04-15',
+        filter_limit: 10,
+        filter_offset: 10,
+        filter_sort_order: 'asc',
+        idSite: '42',
+        method: 'Live.getLastVisitsDetails',
+        period: 'day'
+    });
+    // Should process all events from both pages
+    // 15 visits total × 3 actionDetails each = 45 events
+    expect(result).toMatchObject({
+        date: '2023-04-15',
+        pages: 2,
+        visitsFetched: 15,
+        eventsImported: 45
+    });
+    // Verify database queries: 1 count query + (45 events × 1 query per event)
+    // Note: Each event generates 1 database query for insertion
+    expect(queries.length).toEqual(1 + 45);
+}));

package/dist/__tests__/importEvent.test.js

@@ -0,0 +1,9 @@
+import { getEventsFromMatomoVisit } from '../importEvent.js';
+import matomoVisit from './visit.json';
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+test('getEventsFromMatomoVisit: should merge action events', () => {
+    const visits = getEventsFromMatomoVisit(matomoVisit);
+    expect(visits).toMatchSnapshot();
+});

package/dist/__tests__/migration-imports.test.js

@@ -0,0 +1,38 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { readdir, readFile } from 'fs/promises';
+import { join } from 'path';
+describe('Migration Import Validation', () => {
+    it('should not have src/ imports in migration files', () => __awaiter(void 0, void 0, void 0, function* () {
+        const migrationsDir = join(__dirname, '../migrations');
+        const migrationFiles = yield readdir(migrationsDir);
+        const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
+        const problematicFiles = [];
+        for (const file of tsFiles) {
+            const filePath = join(migrationsDir, file);
+            const content = yield readFile(filePath, 'utf-8');
+            // Check for imports from 'src/' paths
+            const srcImportRegex = /(?:import\s+[^;]+from\s+['"]src\/|require\s*\(\s*['"]src\/)/g;
+            const matches = content.match(srcImportRegex);
+            if (matches) {
+                problematicFiles.push(`${file}: ${matches.join(', ')}`);
+            }
+        }
+        if (problematicFiles.length > 0) {
+            fail(`Migration files contain problematic src/ imports that will fail in production:\n${problematicFiles.join('\n')}\n\nMigration files should use direct constants or relative imports instead.`);
+        }
+    }));
+    it('should have at least one migration file', () => __awaiter(void 0, void 0, void 0, function* () {
+        const migrationsDir = join(__dirname, '../migrations');
+        const migrationFiles = yield readdir(migrationsDir);
+        const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
+        expect(tsFiles.length).toBeGreaterThan(0);
+    }));
+});

package/dist/__tests__/run.test.js

@@ -0,0 +1,95 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+process.env.MATOMO_SITE = '42';
+process.env.PROJECT_NAME = 'some-project';
+process.env.RESULTPERPAGE = '10';
+delete process.env.INITIAL_OFFSET;
+delete process.env.DESTINATION_TABLE;
+delete process.env.STARTDATE;
+// Clear STARTDATE to avoid conflicts with fake timers
+const TEST_DATE = new Date(2023, 3, 1);
+let queries = [];
+let piwikApiCalls = [];
+const result = {
+    command: 'string',
+    rowCount: 0
+};
+jest.mock('pg', () => {
+    const client = {
+        query: (query, values) => {
+            queries.push([query, values]);
+            return result;
+        },
+        release: jest.fn()
+    };
+    const methods = {
+        connect: () => client,
+        on: jest.fn(),
+        query: jest.fn()
+    };
+    return { Pool: jest.fn(() => methods) };
+});
+jest.mock('../PiwikClient', () => {
+    class PiwikMock {
+        constructor(options) {
+            this.options = options;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
+        api(options, cb) {
+            return __awaiter(this, void 0, void 0, function* () {
+                // Import the visit data dynamically to avoid circular dependency
+                const { default: matomoVisit } = yield import('./visit.json');
+                const matomoVisits = [
+                    Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
+                    Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
+                ];
+                piwikApiCalls.push(options);
+                cb(null, matomoVisits);
+            });
+        }
+    }
+    return PiwikMock;
+});
+// Import after mocks are set up
+import run from '../index';
+beforeEach(() => {
+    queries = [];
+    piwikApiCalls = [];
+});
+afterEach(() => {
+    jest.clearAllMocks();
+});
+test('run: should fetch the latest 5 days on matomo', () => __awaiter(void 0, void 0, void 0, function* () {
+    jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+    yield run();
+    expect(piwikApiCalls).toMatchSnapshot();
+}));
+test('run: should fetch the latest event date if no date provided', () => __awaiter(void 0, void 0, void 0, function* () {
+    jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+    yield run();
+    expect(queries[0]).toMatchSnapshot();
+}));
+test('run: should run based on existing data if any', () => __awaiter(void 0, void 0, void 0, function* () {
+    // ensure we use the latest entry in DB
+    expect(1).toEqual(1);
+}));
+test('run: should run SQL queries', () => __awaiter(void 0, void 0, void 0, function* () {
+    jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
+    const result = yield run();
+    expect(queries).toMatchSnapshot();
+    // Updated expectation based on actual behavior with INITIAL_OFFSET=3 (5 days total: 3 days before + today + 1 day after)
+    // 5 days * (6 events per day + 1 count query per day)
+    // Note: We also capture the initial "findLastEventInMatomo" query.
+    expect(queries.length).toEqual(1 + 5 * (6 + 1));
+    expect(result).toMatchObject({
+        daysProcessed: 5,
+        eventsImportedTotal: 5 * 6
+    });
+}));

package/dist/__tests__/visit.json

@@ -0,0 +1,104 @@
+{
+    "idVisit": "124",
+    "idSite": "42",
+    "country": "Argentine",
+    "operatingSystemName": "Mac",
+    "deviceBrand": "Inconnu",
+    "deviceModel": "Générique Bureau",
+    "visitDuration": "300",
+    "daysSinceFirstVisit": "23",
+    "actions": "2",
+    "visitorType": "returningCustomer",
+    "visitorId": "visitorId",
+    "referrerType": "referrerType",
+    "referrerName": "referrerName",
+    "siteName": "tests",
+    "userId": "24",
+    "region": "Buenos Aires",
+    "city": "Buenos Aires",
+    "resolution": "1920x1080",
+    "dimension1": "guest",
+    "dimension3": "page",
+    "dimension6": "shop",
+    "dimension7": "v1.2.3",
+    "dimension8": "fr",
+    "dimension9": "light",
+    "dimension10": "36",
+    "firstActionTimestamp": 1629496512,
+    "actionDetails": [
+        {
+            "type": "event",
+            "url": "https://dive-shop.net/products/basic-wetsuit/",
+            "pageIdAction": "304",
+            "idpageview": "",
+            "serverTimePretty": "20 août 2021 21:35:18",
+            "pageId": "19696671",
+            "eventCategory": "Ecommerce",
+            "eventAction": "Cart change",
+            "timeSpent": 48,
+            "timeSpentPretty": "48s",
+            "pageviewPosition": "8",
+            "timestamp": 1629495318,
+            "icon": "plugins/Morpheus/images/event.png",
+            "iconSVG": "plugins/Morpheus/images/event.svg",
+            "title": "Evènement",
+            "subtitle": "Catégorie: \"Ecommerce', Action: \"Cart change\"",
+            "eventName": "added - Basic Wetsuit",
+            "eventValue": 1,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving",
+            "customVariables": {
+                "1": {
+                    "customVariableName1": "page-author",
+                    "customVariableValue1": "Julien"
+                },
+                "2": {
+                    "customVariableName2": "post-age",
+                    "customVariableValue2": "-430 days"
+                }
+            }
+        },
+        {
+            "type": "action",
+            "url": "https://dive-shop.net/products/diving-boots/",
+            "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
+            "pageIdAction": "60",
+            "idpageview": "8CDIez",
+            "serverTimePretty": "20 août 2021 21:30:25",
+            "pageId": "19696664",
+            "timeSpent": "2",
+            "timeSpentPretty": "2s",
+            "pageviewPosition": "5",
+            "title": "Divezone Brand Diving Boots - Divezone Store",
+            "subtitle": "https://dive-shop.net/products/diving-boots/",
+            "icon": "",
+            "iconSVG": "plugins/Morpheus/images/action.svg",
+            "timestamp": 1629495025,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving"
+        },
+        {
+            "type": "search",
+            "url": "https://dive-shop.net/products/diving-boots/",
+            "pageTitle": "Divezone Brand Diving Boots - Divezone Store",
+            "pageIdAction": "60",
+            "idpageview": "8CDIez",
+            "serverTimePretty": "20 août 2021 21:30:25",
+            "pageId": "19696664",
+            "timeSpent": "2",
+            "timeSpentPretty": "2s",
+            "pageviewPosition": "5",
+            "title": "Divezone Brand Diving Boots - Divezone Store",
+            "subtitle": "https://dive-shop.net/products/diving-boots/",
+            "icon": "",
+            "iconSVG": "plugins/Morpheus/images/action.svg",
+            "timestamp": 1629495022,
+            "dimension2": "julien",
+            "dimension4": "indonesia",
+            "dimension5": "diving",
+            "siteSearchKeyword": "scuba"
+        }
+    ]
+}

package/dist/config.js

@@ -0,0 +1,12 @@
+export const MATOMO_KEY = process.env.MATOMO_KEY || '';
+export const MATOMO_URL = process.env.MATOMO_URL || 'https://matomo.fabrique.social.gouv.fr/';
+export const MATOMO_SITE = process.env.MATOMO_SITE || 0;
+export const PGDATABASE = process.env.PGDATABASE || '';
+export const INITIAL_OFFSET = process.env.INITIAL_OFFSET || '3';
+export const RESULTPERPAGE = process.env.RESULTPERPAGE || '500';
+export const FORCE_STARTDATE = process.env.FORCE_STARTDATE === 'true';
+// We will create both a normal and a partitioned table (MATOMO_TABLE_NAME and PARTITIONED_MATOMO_TABLE_NAME)
+// and use DESTINATION_TABLE to determine which one to write to.
+export const DESTINATION_TABLE = process.env.DESTINATION_TABLE || 'matomo';
+export const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
+export const PARTITIONED_MATOMO_TABLE_NAME = process.env.PARTITIONED_MATOMO_TABLE_NAME || 'matomo_partitioned';

package/dist/db.js

@@ -0,0 +1,31 @@
+import startDebug from 'debug';
+import { Kysely, PostgresDialect } from 'kysely';
+import pkg from 'pg';
+const { Pool } = pkg;
+import { PGDATABASE } from './config.js';
+startDebug('db');
+export const pool = new Pool({
+    connectionString: PGDATABASE,
+    ssl: {
+        rejectUnauthorized: false
+    }
+});
+// Validate pool is properly initialized
+if (!pool || typeof pool.connect !== 'function') {
+    throw new Error('Failed to initialize PostgreSQL connection pool');
+}
+export const db = new Kysely({
+    dialect: new PostgresDialect({
+        pool: pool
+    }),
+    log(event) {
+        if (event.level === 'query') {
+            // debug(event.query.sql)
+            // debug(event.query.parameters)
+        }
+    }
+});
+// Validate the Kysely instance
+if (!db) {
+    throw new Error('Failed to initialize Kysely database instance');
+}

package/dist/importDate.js

@@ -0,0 +1,101 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { formatISO } from 'date-fns';
+import startDebug from 'debug';
+import { sql } from 'kysely';
+import pAll from 'p-all';
+import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
+import { db, pool } from './db.js';
+import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
+const debug = startDebug('importDate');
+/** return date as ISO yyyy-mm-dd */
+const isoDate = (date) => formatISO(date, { representation: 'date' });
+/** check how many visits complete for a given date */
+const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!pool || typeof pool.connect !== 'function') {
+        throw new Error('Database connection pool is invalid or undefined in getRecordsCount');
+    }
+    const result = yield db
+        .selectFrom(DESTINATION_TABLE)
+        .select(db.fn.count('idvisit').distinct().as('count'))
+        // UTC to be iso with matomo matomo data
+        .where(sql `date(timezone('UTC', action_timestamp))`, '=', date)
+        .executeTakeFirst();
+    // start at previous visit in case action didnt finished to record
+    const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
+    return count;
+});
+/** import all event from givent date */
+export const importDate = (piwikApi_1, date_1, ...args_1) => __awaiter(void 0, [piwikApi_1, date_1, ...args_1], void 0, function* (piwikApi, date, filterOffset = 0) {
+    const limit = parseInt(RESULTPERPAGE);
+    // Guard against misconfiguration that can cause infinite pagination loops
+    // (e.g. limit=NaN makes `visits.length < limit` always false).
+    if (!Number.isFinite(limit) || limit <= 0) {
+        throw new Error(`Invalid RESULTPERPAGE: expected a positive integer, got '${RESULTPERPAGE}'`);
+    }
+    const dateIso = isoDate(date);
+    let offset = filterOffset || (yield getRecordsCount(dateIso));
+    if (!Number.isFinite(offset) || offset < 0) {
+        offset = 0;
+    }
+    let pages = 0;
+    let visitsFetched = 0;
+    let eventsImported = 0;
+    while (true) {
+        pages += 1;
+        if (!offset) {
+            debug(`${dateIso}: load ${limit} visits`);
+        }
+        else {
+            debug(`${dateIso}: load ${limit} more visits after ${offset}`);
+        }
+        // fetch visits details
+        const visits = yield new Promise((resolve, reject) => piwikApi({
+            method: 'Live.getLastVisitsDetails',
+            period: 'day',
+            date: dateIso,
+            // minTimestamp: isoDate(new Date()) === isoDate(date) ? date.getTime() / 1000 : undefined, // if today, dont go further (??)
+            filter_limit: limit,
+            filter_offset: offset,
+            filter_sort_order: 'asc',
+            idSite: MATOMO_SITE
+        }, (err, visits = []) => {
+            if (err) {
+                return reject(err);
+            }
+            return resolve(visits);
+        }));
+        visitsFetched += visits.length;
+        debug(`fetched ${visits.length} visits`);
+        // flatten all events
+        const allEvents = visits.flatMap(getEventsFromMatomoVisit);
+        if (allEvents.length) {
+            debug(`import ${allEvents.length} events`);
+            // import events into PG
+            yield pAll(allEvents.map((event) => () => importEvent(event)), { concurrency: 10, stopOnError: true });
+            eventsImported += allEvents.length;
+        }
+        else {
+            debug(`no events to import on this page (${dateIso}, offset ${offset})`);
+        }
+        // stop if we didn't fetch a full page
+        if (visits.length < limit) {
+            break;
+        }
+        offset += limit;
+    }
+    debug(`finished importing ${dateIso}, pages ${pages}, visits ${visitsFetched}`);
+    return {
+        date: dateIso,
+        pages,
+        visitsFetched,
+        eventsImported
+    };
+});