@socialgouv/matomo-postgres 2.1.0 → 2.2.0

package/dist/index.js

@@ -12,55 +12,89 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+const eachDayOfInterval_1 = __importDefault(require("date-fns/eachDayOfInterval"));
+const debug_1 = __importDefault(require("debug"));
 const kysely_1 = require("kysely");
 const p_all_1 = __importDefault(require("p-all"));
-const debug_1 = __importDefault(require("debug"));
-const eachDayOfInterval_1 = __importDefault(require("date-fns/eachDayOfInterval"));
-const PiwikClient_1 = __importDefault(require("./PiwikClient"));
-const db_1 = require("./db");
 const config_1 = require("./config");
+const db_1 = require("./db");
 const importDate_1 = require("./importDate");
-const debug = (0, debug_1.default)("index");
+const PiwikClient_1 = __importDefault(require("./PiwikClient"));
+const debug = (0, debug_1.default)('index');
 function run(date) {
     return __awaiter(this, void 0, void 0, function* () {
-        debug("run, date=" + date);
+        console.log(`🚀 Starting data import process`);
+        debug('run, date=' + date);
+        console.log(`🔗 Initializing Matomo client`);
+        console.log(`  - Matomo URL: ${config_1.MATOMO_URL}`);
+        console.log(`  - Matomo Site ID: ${config_1.MATOMO_SITE}`);
         const piwik = new PiwikClient_1.default(config_1.MATOMO_URL, config_1.MATOMO_KEY);
+        console.log(`📅 Determining reference date for import...`);
         // priority:
         //  - optional parameter date
         //  - last event in the table
         //  - optional env.STARTDATE
         //  - today
         let referenceDate;
-        if (!referenceDate && date)
+        if (!referenceDate && date) {
             referenceDate = new Date(date);
-        if (!referenceDate)
-            referenceDate = yield findLastEventInMatomo(db_1.db);
-        if (!referenceDate && process.env.STARTDATE)
+            console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
+        }
+        if (!referenceDate && process.env.STARTDATE) {
             referenceDate = new Date(process.env.STARTDATE);
-        if (!referenceDate)
+            console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
+        }
+        if (!referenceDate) {
+            console.log(`🔍 Looking for last event in database...`);
+            referenceDate = yield findLastEventInMatomo(db_1.db);
+            if (referenceDate) {
+                console.log(`✅ Found last event, starting from: ${referenceDate.toISOString()}`);
+            }
+            else {
+                console.log(`ℹ️ No previous events found in database`);
+            }
+        }
+        if (!referenceDate) {
             referenceDate = new Date(new Date().getTime() - +config_1.INITIAL_OFFSET * 24 * 60 * 60 * 1000);
+            console.log(`✅ Using default offset (${config_1.INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
+        }
+        const endDate = new Date(new Date().getTime() + 24 * 60 * 60 * 1000);
         const dates = (0, eachDayOfInterval_1.default)({
             start: referenceDate,
-            end: new Date(new Date().getTime() + 24 * 60 * 60 * 1000),
+            end: endDate
         });
+        console.log(`📊 Import date range determined:`);
+        console.log(`  - Start date: ${dates[0].toISOString()}`);
+        console.log(`  - End date: ${endDate.toISOString()}`);
+        console.log(`  - Total days to process: ${dates.length}`);
         debug(`import starting at : ${dates[0].toISOString()}`);
+        console.log(`🔄 Starting sequential import for each date...`);
         // for each date, serial-import data
-        const res = yield (0, p_all_1.default)(dates.map((date) => () => (0, importDate_1.importDate)(piwik.api.bind(piwik), date)), { concurrency: 1, stopOnError: true });
-        debug("close");
+        const res = yield (0, p_all_1.default)(dates.map((date, index) => () => {
+            console.log(`📅 Processing date ${index + 1}/${dates.length}: ${date.toISOString().split('T')[0]}`);
+            return (0, importDate_1.importDate)(piwik.api.bind(piwik), date);
+        }), { concurrency: 1, stopOnError: true });
+        const totalEvents = res.flat().length;
+        console.log(`✅ Import process completed`);
+        console.log(`📈 Summary:`);
+        console.log(`  - Days processed: ${dates.length}`);
+        console.log(`  - Total events imported: ${totalEvents}`);
+        debug('close');
         return res;
     });
 }
 exports.default = run;
 if (require.main === module) {
+    ;
     (() => __awaiter(void 0, void 0, void 0, function* () {
         if (!config_1.MATOMO_SITE)
-            return console.error("Missing env MATOMO_SITE");
+            return console.error('Missing env MATOMO_SITE');
         if (!config_1.MATOMO_KEY)
-            return console.error("Missing env MATOMO_KEY");
+            return console.error('Missing env MATOMO_KEY');
         if (!config_1.PGDATABASE)
-            return console.error("Missing env PGDATABASE");
+            return console.error('Missing env PGDATABASE');
         yield run();
-        debug("run finished");
+        debug('run finished');
         db_1.db.destroy();
     }))();
 }
@@ -68,12 +102,12 @@ function findLastEventInMatomo(db) {
     return __awaiter(this, void 0, void 0, function* () {
         const latest = yield db
             .selectFrom(config_1.DESTINATION_TABLE)
-            .select((0, kysely_1.sql) `action_timestamp at time zone 'UTC'`.as("action_timestamp"))
-            .orderBy("action_timestamp", "desc")
+            .select((0, kysely_1.sql) `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
+            .orderBy('action_timestamp', 'desc')
             .limit(1)
             .executeTakeFirst();
         if (latest) {
-            // check from the day before just to be sure we haev all events
+            // check from the day before just to be sure we have all events
             const date = new Date(new Date(latest.action_timestamp).getTime() - 2 * 24 * 60 * 60 * 1000);
             return date;
         }

package/dist/migrate-down.js

@@ -32,9 +32,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const path = __importStar(require("path"));
 const fs_1 = require("fs");
 const kysely_1 = require("kysely");
+const path = __importStar(require("path"));
 const db_1 = require("./db");
 function migrateDown() {
     return __awaiter(this, void 0, void 0, function* () {
@@ -43,20 +43,20 @@ function migrateDown() {
             provider: new kysely_1.FileMigrationProvider({
                 fs: fs_1.promises,
                 path,
-                migrationFolder: __dirname + "/migrations",
-            }),
+                migrationFolder: __dirname + '/migrations'
+            })
         });
         const { error, results } = yield migrator.migrateDown();
         results === null || results === void 0 ? void 0 : results.forEach((it) => {
-            if (it.status === "Success") {
+            if (it.status === 'Success') {
                 console.log(`down migration "${it.migrationName}" was executed successfully`);
             }
-            else if (it.status === "Error") {
+            else if (it.status === 'Error') {
                 console.error(`failed to execute down migration "${it.migrationName}"`);
             }
         });
         if (error) {
-            console.error("failed to down migrate");
+            console.error('failed to down migrate');
             console.error(error);
             process.exit(1);
         }

package/dist/migrate-latest.js

@@ -32,11 +32,11 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-const path = __importStar(require("path"));
 const fs_1 = require("fs");
 const kysely_1 = require("kysely");
-const db_1 = require("./db");
+const path = __importStar(require("path"));
 const config_1 = require("./config");
+const db_1 = require("./db");
 function migrateToLatest() {
     return __awaiter(this, void 0, void 0, function* () {
         const migrator = new kysely_1.Migrator({
@@ -44,29 +44,29 @@ function migrateToLatest() {
             provider: new kysely_1.FileMigrationProvider({
                 fs: fs_1.promises,
                 path,
-                migrationFolder: __dirname + "/migrations",
+                migrationFolder: __dirname + '/migrations'
             }),
             // allow to have mutliple migratable instances in a single schema
-            migrationTableName: `${config_1.DESTINATION_TABLE}_migration`,
-            migrationLockTableName: `${config_1.DESTINATION_TABLE}_migration_lock`,
+            migrationTableName: `${config_1.MATOMO_TABLE_NAME}_migration`,
+            migrationLockTableName: `${config_1.MATOMO_TABLE_NAME}_migration_lock`
         });
         const { error, results } = yield migrator.migrateToLatest();
         results === null || results === void 0 ? void 0 : results.forEach((it) => {
-            if (it.status === "Success") {
+            if (it.status === 'Success') {
                 console.log(`migration "${it.migrationName}" was executed successfully`);
             }
-            else if (it.status === "Error") {
+            else if (it.status === 'Error') {
                 console.error(`failed to execute migration "${it.migrationName}"`);
             }
         });
         if (error) {
-            console.error("failed to migrate");
+            console.error('failed to migrate');
             console.error(error);
             process.exit(1);
         }
         else {
             if (!(results === null || results === void 0 ? void 0 : results.length)) {
-                console.log("No migration to run");
+                console.log('No migration to run');
             }
         }
     });

package/dist/migrations/20230301-01-initial.js

@@ -11,60 +11,60 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.down = exports.up = void 0;
 const kysely_1 = require("kysely");
-const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
 function up(db) {
     return __awaiter(this, void 0, void 0, function* () {
         yield db.schema
-            .createTable(DESTINATION_TABLE)
+            .createTable(MATOMO_TABLE_NAME)
             .ifNotExists()
-            .addColumn("action_id", "text", (col) => col.unique().notNull())
-            .addColumn("idsite", "text")
-            .addColumn("idvisit", "text")
-            .addColumn("actions", "text")
-            .addColumn("country", "text")
-            .addColumn("region", "text")
-            .addColumn("city", "text")
-            .addColumn("operatingsystemname", "text")
-            .addColumn("devicemodel", "text")
-            .addColumn("devicebrand", "text")
-            .addColumn("visitduration", "text")
-            .addColumn("dayssincefirstvisit", "text")
-            .addColumn("visitortype", "text")
-            .addColumn("sitename", "text")
-            .addColumn("userid", "text")
-            .addColumn("serverdateprettyfirstaction", "date")
-            .addColumn("action_type", "text")
-            .addColumn("action_eventcategory", "text")
-            .addColumn("action_eventaction", "text")
-            .addColumn("action_eventname", "text")
-            .addColumn("action_eventvalue", "numeric")
-            .addColumn("action_timespent", "text")
-            .addColumn("action_timestamp", "timestamptz", (col) => col.defaultTo((0, kysely_1.sql) `now()`))
-            .addColumn("usercustomproperties", "json")
-            .addColumn("usercustomdimensions", "json")
-            .addColumn("dimension1", "text")
-            .addColumn("dimension2", "text")
-            .addColumn("dimension3", "text")
-            .addColumn("dimension4", "text")
-            .addColumn("dimension5", "text")
-            .addColumn("dimension6", "text")
-            .addColumn("dimension7", "text")
-            .addColumn("dimension8", "text")
-            .addColumn("dimension9", "text")
-            .addColumn("dimension10", "text")
-            .addColumn("action_url", "text")
-            .addColumn("sitesearchkeyword", "text")
-            .addColumn("action_title", "text")
-            .addColumn("visitorid", "text")
-            .addColumn("referrertype", "text")
-            .addColumn("referrername", "text")
+            .addColumn('action_id', 'text', (col) => col.unique().notNull())
+            .addColumn('idsite', 'text')
+            .addColumn('idvisit', 'text')
+            .addColumn('actions', 'text')
+            .addColumn('country', 'text')
+            .addColumn('region', 'text')
+            .addColumn('city', 'text')
+            .addColumn('operatingsystemname', 'text')
+            .addColumn('devicemodel', 'text')
+            .addColumn('devicebrand', 'text')
+            .addColumn('visitduration', 'text')
+            .addColumn('dayssincefirstvisit', 'text')
+            .addColumn('visitortype', 'text')
+            .addColumn('sitename', 'text')
+            .addColumn('userid', 'text')
+            .addColumn('serverdateprettyfirstaction', 'date')
+            .addColumn('action_type', 'text')
+            .addColumn('action_eventcategory', 'text')
+            .addColumn('action_eventaction', 'text')
+            .addColumn('action_eventname', 'text')
+            .addColumn('action_eventvalue', 'numeric')
+            .addColumn('action_timespent', 'text')
+            .addColumn('action_timestamp', 'timestamptz', (col) => col.defaultTo((0, kysely_1.sql) `now()`))
+            .addColumn('usercustomproperties', 'json')
+            .addColumn('usercustomdimensions', 'json')
+            .addColumn('dimension1', 'text')
+            .addColumn('dimension2', 'text')
+            .addColumn('dimension3', 'text')
+            .addColumn('dimension4', 'text')
+            .addColumn('dimension5', 'text')
+            .addColumn('dimension6', 'text')
+            .addColumn('dimension7', 'text')
+            .addColumn('dimension8', 'text')
+            .addColumn('dimension9', 'text')
+            .addColumn('dimension10', 'text')
+            .addColumn('action_url', 'text')
+            .addColumn('sitesearchkeyword', 'text')
+            .addColumn('action_title', 'text')
+            .addColumn('visitorid', 'text')
+            .addColumn('referrertype', 'text')
+            .addColumn('referrername', 'text')
             .execute();
     });
 }
 exports.up = up;
 function down(db) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield db.schema.dropTable(DESTINATION_TABLE).execute();
+        yield db.schema.dropTable(MATOMO_TABLE_NAME).execute();
     });
 }
 exports.down = down;

package/dist/migrations/20230301-02-indexes.js

@@ -11,68 +11,68 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.down = exports.up = void 0;
 const kysely_1 = require("kysely");
-const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
 const indexes = [
     {
-        name: "idx_action_eventaction_matomo",
-        columns: ["action_eventaction"],
+        name: 'idx_action_eventaction_matomo',
+        columns: ['action_eventaction']
     },
     {
-        name: "idx_action_eventcategory_matomo",
-        columns: ["action_eventcategory"],
+        name: 'idx_action_eventcategory_matomo',
+        columns: ['action_eventcategory']
     },
     {
-        name: "idx_action_id",
-        columns: ["action_id"],
+        name: 'idx_action_id',
+        columns: ['action_id']
     },
     {
-        name: "idx_action_timestamp_matomo",
-        columns: ["action_timestamp"],
+        name: 'idx_action_timestamp_matomo',
+        columns: ['action_timestamp']
     },
     {
-        name: "idx_action_type_matomo",
-        columns: ["action_type"],
+        name: 'idx_action_type_matomo',
+        columns: ['action_type']
     },
     {
-        name: "idx_actionurl",
-        columns: ["action_url"],
+        name: 'idx_actionurl',
+        columns: ['action_url']
     },
     {
-        name: "idx_dimension1",
-        columns: ["dimension1"],
+        name: 'idx_dimension1',
+        columns: ['dimension1']
     },
     {
-        name: "idx_dimension2",
-        columns: ["dimension2"],
+        name: 'idx_dimension2',
+        columns: ['dimension2']
     },
     {
-        name: "idx_dimension3",
-        columns: ["dimension3"],
+        name: 'idx_dimension3',
+        columns: ['dimension3']
     },
     {
-        name: "idx_dimension4",
-        columns: ["dimension4"],
+        name: 'idx_dimension4',
+        columns: ['dimension4']
     },
     {
-        name: "idx_dimension5",
-        columns: ["dimension5"],
+        name: 'idx_dimension5',
+        columns: ['dimension5']
     },
     {
-        name: "idx_idvisit_matomo",
-        columns: ["idvisit"],
+        name: 'idx_idvisit_matomo',
+        columns: ['idvisit']
     },
     {
-        name: "idx_region",
-        columns: ["region"],
+        name: 'idx_region',
+        columns: ['region']
     },
     {
-        name: "idx_userid",
-        columns: ["userid"],
+        name: 'idx_userid',
+        columns: ['userid']
     },
     {
-        name: "idx_visitorid",
-        columns: ["visitorid"],
-    },
+        name: 'idx_visitorid',
+        columns: ['visitorid']
+    }
 ];
 function up(db) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -80,15 +80,15 @@ function up(db) {
             yield db.schema
                 .createIndex(index.name)
                 .ifNotExists()
-                .on(DESTINATION_TABLE)
-                .using("btree")
+                .on(MATOMO_TABLE_NAME)
+                .using('btree')
                 .columns(index.columns)
                 .execute();
         }));
         yield db.schema
-            .createIndex("actions_day")
+            .createIndex('actions_day')
             .ifNotExists()
-            .on(DESTINATION_TABLE)
+            .on(MATOMO_TABLE_NAME)
             .expression((0, kysely_1.sql) `date(timezone('UTC', action_timestamp))`)
             .execute();
     });
@@ -99,7 +99,7 @@ function down(db) {
         indexes.forEach((index) => __awaiter(this, void 0, void 0, function* () {
             yield db.schema.dropIndex(index.name).execute();
         }));
-        db.schema.dropIndex("actions_day").execute();
+        db.schema.dropIndex('actions_day').execute();
     });
 }
 exports.down = down;

package/dist/migrations/20250425-01-add-resolution.js

@@ -10,16 +10,22 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.down = exports.up = void 0;
-const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
+const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
 function up(db) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield db.schema.alterTable(DESTINATION_TABLE).addColumn("resolution", "text").execute();
+        yield db.schema
+            .alterTable(MATOMO_TABLE_NAME)
+            .addColumn('resolution', 'text')
+            .execute();
     });
 }
 exports.up = up;
 function down(db) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield db.schema.alterTable(DESTINATION_TABLE).dropColumn("resolution").execute();
+        yield db.schema
+            .alterTable(MATOMO_TABLE_NAME)
+            .dropColumn('resolution')
+            .execute();
     });
 }
 exports.down = down;