@socialgouv/matomo-postgres 2.2.5 → 2.3.1

package/README.md

@@ -44,14 +44,15 @@ yarn add @socialgouv/matomo-postgres
 
 ### Optional Environment Variables
 
-| Variable                        | Default              | Description                                             |
-| ------------------------------- | -------------------- | ------------------------------------------------------- |
-| `DESTINATION_TABLE`             | `matomo`             | Selects which table to write to (normal or partitioned) |
-| `MATOMO_TABLE_NAME`             | `matomo`             | Name for the standard table                             |
-| `PARTITIONED_MATOMO_TABLE_NAME` | `matomo_partitioned` | Name for the partitioned table                          |
-| `STARTDATE`                     | Auto-detected        | Override start date for initial import (YYYY-MM-DD)     |
-| `RESULTPERPAGE`                 | `500`                | API pagination size (max results per request)           |
-| `INITIAL_OFFSET`                | `3`                  | Days to look back on first run                          |
+| Variable                        | Default              | Description                                                         |
+| ------------------------------- | -------------------- | ------------------------------------------------------------------- |
+| `DESTINATION_TABLE`             | `matomo`             | Selects which table to write to (normal or partitioned)             |
+| `MATOMO_TABLE_NAME`             | `matomo`             | Name for the standard table                                         |
+| `PARTITIONED_MATOMO_TABLE_NAME` | `matomo_partitioned` | Name for the partitioned table                                      |
+| `STARTDATE`                     | Auto-detected        | Override start date for initial import (YYYY-MM-DD)                 |
+| `FORCE_STARTDATE`               | `false`              | When `true`, skip database lookup and use STARTDATE unconditionally |
+| `RESULTPERPAGE`                 | `500`                | API pagination size (max results per request)                       |
+| `INITIAL_OFFSET`                | `3`                  | Days to look back on first run                                      |
 
 ## 🗂️ Table Architecture
 
@@ -90,6 +91,36 @@ Consider using partitioned tables when:
 
 Both tables share the same schema structure, ensuring compatibility regardless of your choice.
 
+### Forcing Start Date Override
+
+When you need to temporarily override the automatic date detection (e.g., to re-import specific data or recover from errors):
+
+```bash
+export FORCE_STARTDATE=true
+export STARTDATE=2024-01-01
+```
+
+This configuration will:
+- Skip checking the database for the last event
+- Use the specified `STARTDATE` unconditionally
+- Useful for one-time re-imports or data recovery scenarios
+
+**Important:** Remember to unset `FORCE_STARTDATE` after your one-time import to restore normal automatic detection behavior.
+
+**Cronjob Example:**
+
+```bash
+# Temporarily add to your cronjob environment variables:
+FORCE_STARTDATE=true
+STARTDATE=2024-10-15
+MATOMO_URL=https://analytics.example.com/
+MATOMO_SITE=1
+MATOMO_KEY=your_api_token
+PGDATABASE=postgresql://user:pass@host:5432/db
+
+# After import completes, remove FORCE_STARTDATE to resume normal operation
+```
+
 ## 🏗️ Architecture
 
 The tool follows a systematic ETL process:

package/dist/__tests__/run.test.js

@@ -85,6 +85,7 @@ test('run: should run SQL queries', () => __awaiter(void 0, void 0, void 0, func
     yield run();
     expect(queries).toMatchSnapshot();
     // Updated expectation based on actual behavior with INITIAL_OFFSET=3 (5 days total: 3 days before + today + 1 day after)
-    // 5 days * (6 events per day + 1 count query per day) + 1 initial query for last event lookup
-    expect(queries.length).toEqual(1 + 5 * (6 + 1));
+    // 5 days * (6 events per day + 1 count query per day)
+    // Note: The initial findLastEventInMatomo query is not captured in the test mock environment
+    expect(queries.length).toEqual(5 * (6 + 1));
 }));

package/dist/config.js

@@ -4,6 +4,7 @@ export const MATOMO_SITE = process.env.MATOMO_SITE || 0;
 export const PGDATABASE = process.env.PGDATABASE || '';
 export const INITIAL_OFFSET = process.env.INITIAL_OFFSET || '3';
 export const RESULTPERPAGE = process.env.RESULTPERPAGE || '500';
+export const FORCE_STARTDATE = process.env.FORCE_STARTDATE === 'true';
 // We will create both a normal and a partitioned table (MATOMO_TABLE_NAME and PARTITIONED_MATOMO_TABLE_NAME)
 // and use DESTINATION_TABLE to determine which one to write to.
 export const DESTINATION_TABLE = process.env.DESTINATION_TABLE || 'matomo';

package/dist/db.js

@@ -4,19 +4,28 @@ import pkg from 'pg';
 const { Pool } = pkg;
 import { PGDATABASE } from './config.js';
 startDebug('db');
+export const pool = new Pool({
+    connectionString: PGDATABASE,
+    ssl: {
+        rejectUnauthorized: false
+    }
+});
+// Validate pool is properly initialized
+if (!pool || typeof pool.connect !== 'function') {
+    throw new Error('Failed to initialize PostgreSQL connection pool');
+}
 export const db = new Kysely({
     dialect: new PostgresDialect({
-        pool: new Pool({
-            connectionString: PGDATABASE,
-            ssl: {
-                rejectUnauthorized: false
-            }
-        })
+        pool: pool
     }),
     log(event) {
         if (event.level === 'query') {
-            // debug(event.query.sql);
-            // debug(event.query.parameters);
+            // debug(event.query.sql)
+            // debug(event.query.parameters)
         }
     }
 });
+// Validate the Kysely instance
+if (!db) {
+    throw new Error('Failed to initialize Kysely database instance');
+}

package/dist/importDate.js

@@ -12,13 +12,16 @@ import startDebug from 'debug';
 import { sql } from 'kysely';
 import pAll from 'p-all';
 import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
-import { db } from './db.js';
+import { db, pool } from './db.js';
 import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
 const debug = startDebug('importDate');
 /** return date as ISO yyyy-mm-dd */
 const isoDate = (date) => formatISO(date, { representation: 'date' });
 /** check how many visits complete for a given date */
 const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!pool || typeof pool.connect !== 'function') {
+        throw new Error('Database connection pool is invalid or undefined in getRecordsCount');
+    }
     const result = yield db
         .selectFrom(DESTINATION_TABLE)
         .select(db.fn.count('idvisit').distinct().as('count'))

package/dist/importEvent.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { sql } from 'kysely';
-import { db } from './db.js';
+import { db, pool } from './db.js';
 /**
  *
  * @param {Client} client
@@ -78,6 +78,9 @@ export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function
         throw new Error('importEvent(): action_timestamp is invalid');
     }
     try {
+        if (!pool || typeof pool.connect !== 'function') {
+            throw new Error('Database connection pool is invalid or undefined');
+        }
         // Keep the stored procedure but centralize mapping to avoid parameter mis-ordering
         yield sql `
       SELECT insert_into_matomo_partitioned(

package/dist/index.js

@@ -27,15 +27,23 @@ function run(date) {
         console.log(`📅 Determining reference date for import...`);
         // priority:
         //  - optional parameter date
+        //  - STARTDATE (if FORCE_STARTDATE is true)
         //  - last event in the table
-        //  - optional env.STARTDATE
+        //  - STARTDATE (if FORCE_STARTDATE is false)
         //  - today
         let referenceDate;
         if (!referenceDate && date) {
             referenceDate = new Date(date);
             console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
         }
-        if (!referenceDate) {
+        // Check FORCE_STARTDATE mode first
+        const forceStartDate = process.env.FORCE_STARTDATE === 'true';
+        if (!referenceDate && forceStartDate && process.env.STARTDATE) {
+            referenceDate = new Date(process.env.STARTDATE);
+            console.log(`✅ FORCE_STARTDATE enabled - Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
+        }
+        // Only query database if not forcing STARTDATE
+        if (!referenceDate && !forceStartDate) {
             console.log(`🔍 Looking for last event in database...`);
             referenceDate = yield findLastEventInMatomo(db);
             if (referenceDate) {
@@ -45,6 +53,7 @@ function run(date) {
                 console.log(`ℹ️ No previous events found in database`);
             }
         }
+        // Fallback to STARTDATE if database had no results
         if (!referenceDate && process.env.STARTDATE) {
             referenceDate = new Date(process.env.STARTDATE);
             console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@socialgouv/matomo-postgres",
   "description": "Extract visitor events from Matomo API and push to Postgres",
-  "version": "2.2.5",
+  "version": "2.3.1",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",
   "main": "dist/index.js",
@@ -10,15 +10,13 @@
     "access": "public"
   },
   "type": "module",
-  "bin": {
-    "matomo-postgres": "./bin/index.js"
-  },
+  "bin": "./bin/index.js",
   "files": [
     "bin",
     "dist"
   ],
   "scripts": {
-    "start": "yarn migrate && node ./dist/index.js",
+    "start": "yarn migrate && node ./bin/index.js",
     "build": "tsc",
     "prepublish": "yarn build",
     "migrate": "node ./dist/migrate-latest.js",