@socialgouv/matomo-postgres 2.2.4 → 2.3.0

package/README.md

@@ -44,14 +44,15 @@ yarn add @socialgouv/matomo-postgres
 
 ### Optional Environment Variables
 
-| Variable                        | Default              | Description                                             |
-| ------------------------------- | -------------------- | ------------------------------------------------------- |
-| `DESTINATION_TABLE`             | `matomo`             | Selects which table to write to (normal or partitioned) |
-| `MATOMO_TABLE_NAME`             | `matomo`             | Name for the standard table                             |
-| `PARTITIONED_MATOMO_TABLE_NAME` | `matomo_partitioned` | Name for the partitioned table                          |
-| `STARTDATE`                     | Auto-detected        | Override start date for initial import (YYYY-MM-DD)     |
-| `RESULTPERPAGE`                 | `500`                | API pagination size (max results per request)           |
-| `INITIAL_OFFSET`                | `3`                  | Days to look back on first run                          |
+| Variable                        | Default              | Description                                                         |
+| ------------------------------- | -------------------- | ------------------------------------------------------------------- |
+| `DESTINATION_TABLE`             | `matomo`             | Selects which table to write to (normal or partitioned)             |
+| `MATOMO_TABLE_NAME`             | `matomo`             | Name for the standard table                                         |
+| `PARTITIONED_MATOMO_TABLE_NAME` | `matomo_partitioned` | Name for the partitioned table                                      |
+| `STARTDATE`                     | Auto-detected        | Override start date for initial import (YYYY-MM-DD)                 |
+| `FORCE_STARTDATE`               | `false`              | When `true`, skip database lookup and use STARTDATE unconditionally |
+| `RESULTPERPAGE`                 | `500`                | API pagination size (max results per request)                       |
+| `INITIAL_OFFSET`                | `3`                  | Days to look back on first run                                      |
 
 ## 🗂️ Table Architecture
 
@@ -90,6 +91,36 @@ Consider using partitioned tables when:
 
 Both tables share the same schema structure, ensuring compatibility regardless of your choice.
 
+### Forcing Start Date Override
+
+When you need to temporarily override the automatic date detection (e.g., to re-import specific data or recover from errors):
+
+```bash
+export FORCE_STARTDATE=true
+export STARTDATE=2024-01-01
+```
+
+This configuration will:
+- Skip checking the database for the last event
+- Use the specified `STARTDATE` unconditionally
+- Useful for one-time re-imports or data recovery scenarios
+
+**Important:** Remember to unset `FORCE_STARTDATE` after your one-time import to restore normal automatic detection behavior.
+
+**Cronjob Example:**
+
+```bash
+# Temporarily add to your cronjob environment variables:
+FORCE_STARTDATE=true
+STARTDATE=2024-10-15
+MATOMO_URL=https://analytics.example.com/
+MATOMO_SITE=1
+MATOMO_KEY=your_api_token
+PGDATABASE=postgresql://user:pass@host:5432/db
+
+# After import completes, remove FORCE_STARTDATE to resume normal operation
+```
+
 ## 🏗️ Architecture
 
 The tool follows a systematic ETL process:

package/dist/__tests__/migration-imports.test.js

@@ -0,0 +1,38 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { readdir, readFile } from 'fs/promises';
+import { join } from 'path';
+describe('Migration Import Validation', () => {
+    it('should not have src/ imports in migration files', () => __awaiter(void 0, void 0, void 0, function* () {
+        const migrationsDir = join(__dirname, '../migrations');
+        const migrationFiles = yield readdir(migrationsDir);
+        const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
+        const problematicFiles = [];
+        for (const file of tsFiles) {
+            const filePath = join(migrationsDir, file);
+            const content = yield readFile(filePath, 'utf-8');
+            // Check for imports from 'src/' paths
+            const srcImportRegex = /(?:import\s+[^;]+from\s+['"]src\/|require\s*\(\s*['"]src\/)/g;
+            const matches = content.match(srcImportRegex);
+            if (matches) {
+                problematicFiles.push(`${file}: ${matches.join(', ')}`);
+            }
+        }
+        if (problematicFiles.length > 0) {
+            fail(`Migration files contain problematic src/ imports that will fail in production:\n${problematicFiles.join('\n')}\n\nMigration files should use direct constants or relative imports instead.`);
+        }
+    }));
+    it('should have at least one migration file', () => __awaiter(void 0, void 0, void 0, function* () {
+        const migrationsDir = join(__dirname, '../migrations');
+        const migrationFiles = yield readdir(migrationsDir);
+        const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
+        expect(tsFiles.length).toBeGreaterThan(0);
+    }));
+});

package/dist/config.js

@@ -4,6 +4,7 @@ export const MATOMO_SITE = process.env.MATOMO_SITE || 0;
 export const PGDATABASE = process.env.PGDATABASE || '';
 export const INITIAL_OFFSET = process.env.INITIAL_OFFSET || '3';
 export const RESULTPERPAGE = process.env.RESULTPERPAGE || '500';
+export const FORCE_STARTDATE = process.env.FORCE_STARTDATE === 'true';
 // We will create both a normal and a partitioned table (MATOMO_TABLE_NAME and PARTITIONED_MATOMO_TABLE_NAME)
 // and use DESTINATION_TABLE to determine which one to write to.
 export const DESTINATION_TABLE = process.env.DESTINATION_TABLE || 'matomo';

package/dist/index.js

@@ -27,15 +27,23 @@ function run(date) {
         console.log(`📅 Determining reference date for import...`);
         // priority:
         //  - optional parameter date
+        //  - STARTDATE (if FORCE_STARTDATE is true)
         //  - last event in the table
-        //  - optional env.STARTDATE
+        //  - STARTDATE (if FORCE_STARTDATE is false)
         //  - today
         let referenceDate;
         if (!referenceDate && date) {
             referenceDate = new Date(date);
             console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
         }
-        if (!referenceDate) {
+        // Check FORCE_STARTDATE mode first
+        const forceStartDate = process.env.FORCE_STARTDATE === 'true';
+        if (!referenceDate && forceStartDate && process.env.STARTDATE) {
+            referenceDate = new Date(process.env.STARTDATE);
+            console.log(`✅ FORCE_STARTDATE enabled - Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
+        }
+        // Only query database if not forcing STARTDATE
+        if (!referenceDate && !forceStartDate) {
             console.log(`🔍 Looking for last event in database...`);
             referenceDate = yield findLastEventInMatomo(db);
             if (referenceDate) {
@@ -45,6 +53,7 @@ function run(date) {
                 console.log(`ℹ️ No previous events found in database`);
             }
         }
+        // Fallback to STARTDATE if database had no results
         if (!referenceDate && process.env.STARTDATE) {
             referenceDate = new Date(process.env.STARTDATE);
             console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);

package/dist/migrations/20250715-01-weekly-partitioning.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { sql } from 'kysely';
-import { PARTITIONED_MATOMO_TABLE_NAME } from 'src/config';
+import { PARTITIONED_MATOMO_TABLE_NAME } from '../config.js';
 export function up(db) {
     return __awaiter(this, void 0, void 0, function* () {
         // First, create the partitioned table structure as a partitioned table

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@socialgouv/matomo-postgres",
   "description": "Extract visitor events from Matomo API and push to Postgres",
-  "version": "2.2.4",
+  "version": "2.3.0",
   "types": "types/index.d.ts",
   "license": "Apache-2.0",
   "main": "dist/index.js",