@socialgouv/matomo-postgres 2.2.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,3 @@
1
- "use strict";
2
1
  var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
2
  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
3
  return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,36 +7,31 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
8
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
9
8
  });
10
9
  };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- exports.importDate = void 0;
16
- const formatISO_1 = __importDefault(require("date-fns/formatISO"));
17
- const debug_1 = __importDefault(require("debug"));
18
- const kysely_1 = require("kysely");
19
- const p_all_1 = __importDefault(require("p-all"));
20
- const config_1 = require("./config");
21
- const db_1 = require("./db");
22
- const importEvent_1 = require("./importEvent");
23
- const debug = (0, debug_1.default)('importDate');
10
+ import { formatISO } from 'date-fns';
11
+ import startDebug from 'debug';
12
+ import { sql } from 'kysely';
13
+ import pAll from 'p-all';
14
+ import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
15
+ import { db } from './db.js';
16
+ import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
17
+ const debug = startDebug('importDate');
24
18
  /** return date as ISO yyyy-mm-dd */
25
- const isoDate = (date) => (0, formatISO_1.default)(date, { representation: 'date' });
19
+ const isoDate = (date) => formatISO(date, { representation: 'date' });
26
20
  /** check how many visits complete for a given date */
27
21
  const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
28
- const result = yield db_1.db
29
- .selectFrom(config_1.DESTINATION_TABLE)
30
- .select(db_1.db.fn.count('idvisit').distinct().as('count'))
22
+ const result = yield db
23
+ .selectFrom(DESTINATION_TABLE)
24
+ .select(db.fn.count('idvisit').distinct().as('count'))
31
25
  // UTC to be iso with matomo matomo data
32
- .where((0, kysely_1.sql) `date(timezone('UTC', action_timestamp))`, '=', date)
26
+ .where(sql `date(timezone('UTC', action_timestamp))`, '=', date)
33
27
  .executeTakeFirst();
34
28
  // start at previous visit in case action didnt finished to record
35
29
  const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
36
30
  return count;
37
31
  });
38
32
  /** import all event from givent date */
39
- const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void 0, void 0, function* () {
40
- const limit = parseInt(config_1.RESULTPERPAGE);
33
+ export const importDate = (piwikApi_1, date_1, ...args_1) => __awaiter(void 0, [piwikApi_1, date_1, ...args_1], void 0, function* (piwikApi, date, filterOffset = 0) {
34
+ const limit = parseInt(RESULTPERPAGE);
41
35
  const offset = filterOffset || (yield getRecordsCount(isoDate(date)));
42
36
  if (!offset) {
43
37
  debug(`${isoDate(date)}: load ${limit} visits`);
@@ -54,7 +48,7 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
54
48
  filter_limit: limit,
55
49
  filter_offset: offset,
56
50
  filter_sort_order: 'asc',
57
- idSite: config_1.MATOMO_SITE
51
+ idSite: MATOMO_SITE
58
52
  }, (err, visits = []) => {
59
53
  if (err) {
60
54
  console.error('err', err);
@@ -64,7 +58,7 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
64
58
  }));
65
59
  debug(`fetched ${visits.length} visits`);
66
60
  // flatten all events
67
- const eventsFromVisits = visits.flatMap(importEvent_1.getEventsFromMatomoVisit);
61
+ const eventsFromVisits = visits.flatMap(getEventsFromMatomoVisit);
68
62
  const allEvents = eventsFromVisits.filter((_event) => {
69
63
  return true;
70
64
  });
@@ -74,15 +68,13 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
74
68
  }
75
69
  debug(`import ${allEvents.length} events`);
76
70
  // serial-import events into PG
77
- yield (0, p_all_1.default)(allEvents.map((event) => () => (0, importEvent_1.importEvent)(event)), { concurrency: 10, stopOnError: true });
71
+ yield pAll(allEvents.map((event) => () => importEvent(event)), { concurrency: 10, stopOnError: true });
78
72
  // continue to next page if necessary
79
73
  if (visits.length === limit) {
80
74
  const nextOffset = offset + limit;
81
- const nextEvents = yield (0, exports.importDate)(piwikApi, date, nextOffset);
75
+ const nextEvents = yield importDate(piwikApi, date, nextOffset);
82
76
  return [...allEvents, ...(nextEvents || [])];
83
77
  }
84
78
  debug(`finished importing ${isoDate(date)}, offset ${offset}`);
85
79
  return allEvents;
86
80
  });
87
- exports.importDate = importDate;
88
- module.exports = { importDate: exports.importDate };
@@ -1,4 +1,3 @@
1
- "use strict";
2
1
  var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
2
  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
3
  return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,10 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
8
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
9
8
  });
10
9
  };
11
- Object.defineProperty(exports, "__esModule", { value: true });
12
- exports.getEventsFromMatomoVisit = exports.importEvent = void 0;
13
- const kysely_1 = require("kysely");
14
- const db_1 = require("./db");
10
+ import { sql } from 'kysely';
11
+ import { db } from './db.js';
15
12
  /**
16
13
  *
17
14
  * @param {Client} client
@@ -19,59 +16,130 @@ const db_1 = require("./db");
19
16
  *
20
17
  * @return {Promise<void>}
21
18
  */
22
- const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
19
+ export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
23
20
  var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
24
- // Use the stored procedure for safe insertion with automatic partition creation
25
- yield (0, kysely_1.sql) `
26
- SELECT insert_into_matomo_partitioned(
27
- ${(_a = event.action_id) !== null && _a !== void 0 ? _a : ''},
28
- ${event.action_timestamp ? new Date(event.action_timestamp) : new Date()},
29
- ${(_b = event.idsite) !== null && _b !== void 0 ? _b : ''},
30
- ${(_c = event.idvisit) !== null && _c !== void 0 ? _c : ''},
31
- ${(_d = event.actions) !== null && _d !== void 0 ? _d : null},
32
- ${(_e = event.country) !== null && _e !== void 0 ? _e : null},
33
- ${(_f = event.region) !== null && _f !== void 0 ? _f : null},
34
- ${(_g = event.city) !== null && _g !== void 0 ? _g : null},
35
- ${(_h = event.operatingsystemname) !== null && _h !== void 0 ? _h : null},
36
- ${(_j = event.devicemodel) !== null && _j !== void 0 ? _j : null},
37
- ${(_k = event.devicebrand) !== null && _k !== void 0 ? _k : null},
38
- ${(_l = event.visitduration) !== null && _l !== void 0 ? _l : null},
39
- ${(_m = event.dayssincefirstvisit) !== null && _m !== void 0 ? _m : null},
40
- ${(_o = event.visitortype) !== null && _o !== void 0 ? _o : null},
41
- ${(_p = event.sitename) !== null && _p !== void 0 ? _p : null},
42
- ${(_q = event.userid) !== null && _q !== void 0 ? _q : null},
43
- ${event.serverdateprettyfirstaction
44
- ? new Date(event.serverdateprettyfirstaction)
45
- : null},
46
- ${(_r = event.action_type) !== null && _r !== void 0 ? _r : ''},
47
- ${(_s = event.action_eventcategory) !== null && _s !== void 0 ? _s : ''},
48
- ${(_t = event.action_eventaction) !== null && _t !== void 0 ? _t : ''},
49
- ${(_u = event.action_eventname) !== null && _u !== void 0 ? _u : ''},
50
- ${event.action_eventvalue ? Number(event.action_eventvalue) : 0},
51
- ${(_v = event.action_timespent) !== null && _v !== void 0 ? _v : '0'},
52
- ${(_w = event.usercustomproperties) !== null && _w !== void 0 ? _w : null},
53
- ${(_x = event.usercustomdimensions) !== null && _x !== void 0 ? _x : null},
54
- ${(_y = event.dimension1) !== null && _y !== void 0 ? _y : null},
55
- ${(_z = event.dimension2) !== null && _z !== void 0 ? _z : null},
56
- ${(_0 = event.dimension3) !== null && _0 !== void 0 ? _0 : null},
57
- ${(_1 = event.dimension4) !== null && _1 !== void 0 ? _1 : null},
58
- ${(_2 = event.dimension5) !== null && _2 !== void 0 ? _2 : null},
59
- ${(_3 = event.dimension6) !== null && _3 !== void 0 ? _3 : null},
60
- ${(_4 = event.dimension7) !== null && _4 !== void 0 ? _4 : null},
61
- ${(_5 = event.dimension8) !== null && _5 !== void 0 ? _5 : null},
62
- ${(_6 = event.dimension9) !== null && _6 !== void 0 ? _6 : null},
63
- ${(_7 = event.dimension10) !== null && _7 !== void 0 ? _7 : null},
64
- ${(_8 = event.action_url) !== null && _8 !== void 0 ? _8 : null},
65
- ${(_9 = event.sitesearchkeyword) !== null && _9 !== void 0 ? _9 : null},
66
- ${(_10 = event.action_title) !== null && _10 !== void 0 ? _10 : null},
67
- ${(_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null},
68
- ${(_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null},
69
- ${(_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null},
70
- ${(_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null}
71
- )
72
- `.execute(db_1.db);
21
+ // Build a sanitized, typed data object to reduce drift and ensure defaults in one place
22
+ const eventData = {
23
+ action_id: (_a = event.action_id) !== null && _a !== void 0 ? _a : '',
24
+ action_timestamp: event.action_timestamp
25
+ ? new Date(event.action_timestamp)
26
+ : new Date(),
27
+ idsite: (_b = event.idsite) !== null && _b !== void 0 ? _b : '',
28
+ idvisit: (_c = event.idvisit) !== null && _c !== void 0 ? _c : '',
29
+ actions: (_d = event.actions) !== null && _d !== void 0 ? _d : null,
30
+ country: (_e = event.country) !== null && _e !== void 0 ? _e : null,
31
+ region: (_f = event.region) !== null && _f !== void 0 ? _f : null,
32
+ city: (_g = event.city) !== null && _g !== void 0 ? _g : null,
33
+ operatingsystemname: (_h = event.operatingsystemname) !== null && _h !== void 0 ? _h : null,
34
+ devicemodel: (_j = event.devicemodel) !== null && _j !== void 0 ? _j : null,
35
+ devicebrand: (_k = event.devicebrand) !== null && _k !== void 0 ? _k : null,
36
+ visitduration: (_l = event.visitduration) !== null && _l !== void 0 ? _l : null,
37
+ dayssincefirstvisit: (_m = event.dayssincefirstvisit) !== null && _m !== void 0 ? _m : null,
38
+ visitortype: (_o = event.visitortype) !== null && _o !== void 0 ? _o : null,
39
+ sitename: (_p = event.sitename) !== null && _p !== void 0 ? _p : null,
40
+ userid: (_q = event.userid) !== null && _q !== void 0 ? _q : null,
41
+ serverdateprettyfirstaction: event.serverdateprettyfirstaction
42
+ ? new Date(event.serverdateprettyfirstaction)
43
+ : null,
44
+ action_type: (_r = event.action_type) !== null && _r !== void 0 ? _r : '',
45
+ action_eventcategory: (_s = event.action_eventcategory) !== null && _s !== void 0 ? _s : '',
46
+ action_eventaction: (_t = event.action_eventaction) !== null && _t !== void 0 ? _t : '',
47
+ action_eventname: (_u = event.action_eventname) !== null && _u !== void 0 ? _u : '',
48
+ action_eventvalue: event.action_eventvalue
49
+ ? Number(event.action_eventvalue)
50
+ : 0,
51
+ action_timespent: (_v = event.action_timespent) !== null && _v !== void 0 ? _v : '0',
52
+ usercustomproperties: (_w = event.usercustomproperties) !== null && _w !== void 0 ? _w : null,
53
+ usercustomdimensions: (_x = event.usercustomdimensions) !== null && _x !== void 0 ? _x : null,
54
+ dimension1: (_y = event.dimension1) !== null && _y !== void 0 ? _y : null,
55
+ dimension2: (_z = event.dimension2) !== null && _z !== void 0 ? _z : null,
56
+ dimension3: (_0 = event.dimension3) !== null && _0 !== void 0 ? _0 : null,
57
+ dimension4: (_1 = event.dimension4) !== null && _1 !== void 0 ? _1 : null,
58
+ dimension5: (_2 = event.dimension5) !== null && _2 !== void 0 ? _2 : null,
59
+ dimension6: (_3 = event.dimension6) !== null && _3 !== void 0 ? _3 : null,
60
+ dimension7: (_4 = event.dimension7) !== null && _4 !== void 0 ? _4 : null,
61
+ dimension8: (_5 = event.dimension8) !== null && _5 !== void 0 ? _5 : null,
62
+ dimension9: (_6 = event.dimension9) !== null && _6 !== void 0 ? _6 : null,
63
+ dimension10: (_7 = event.dimension10) !== null && _7 !== void 0 ? _7 : null,
64
+ action_url: (_8 = event.action_url) !== null && _8 !== void 0 ? _8 : null,
65
+ sitesearchkeyword: (_9 = event.sitesearchkeyword) !== null && _9 !== void 0 ? _9 : null,
66
+ action_title: (_10 = event.action_title) !== null && _10 !== void 0 ? _10 : null,
67
+ visitorid: (_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null,
68
+ referrertype: (_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null,
69
+ referrername: (_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null,
70
+ resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null
71
+ };
72
+ // Minimal runtime validation for required fields
73
+ if (!eventData.action_id || eventData.action_id.trim().length === 0) {
74
+ throw new Error('importEvent(): action_id is required and cannot be empty');
75
+ }
76
+ if (!(eventData.action_timestamp instanceof Date) ||
77
+ isNaN(eventData.action_timestamp.getTime())) {
78
+ throw new Error('importEvent(): action_timestamp is invalid');
79
+ }
80
+ try {
81
+ // Keep the stored procedure but centralize mapping to avoid parameter mis-ordering
82
+ yield sql `
83
+ SELECT insert_into_matomo_partitioned(
84
+ ${eventData.action_id},
85
+ ${eventData.action_timestamp},
86
+ ${eventData.idsite},
87
+ ${eventData.idvisit},
88
+ ${eventData.actions},
89
+ ${eventData.country},
90
+ ${eventData.region},
91
+ ${eventData.city},
92
+ ${eventData.operatingsystemname},
93
+ ${eventData.devicemodel},
94
+ ${eventData.devicebrand},
95
+ ${eventData.visitduration},
96
+ ${eventData.dayssincefirstvisit},
97
+ ${eventData.visitortype},
98
+ ${eventData.sitename},
99
+ ${eventData.userid},
100
+ ${eventData.serverdateprettyfirstaction},
101
+ ${eventData.action_type},
102
+ ${eventData.action_eventcategory},
103
+ ${eventData.action_eventaction},
104
+ ${eventData.action_eventname},
105
+ ${eventData.action_eventvalue},
106
+ ${eventData.action_timespent},
107
+ ${eventData.usercustomproperties},
108
+ ${eventData.usercustomdimensions},
109
+ ${eventData.dimension1},
110
+ ${eventData.dimension2},
111
+ ${eventData.dimension3},
112
+ ${eventData.dimension4},
113
+ ${eventData.dimension5},
114
+ ${eventData.dimension6},
115
+ ${eventData.dimension7},
116
+ ${eventData.dimension8},
117
+ ${eventData.dimension9},
118
+ ${eventData.dimension10},
119
+ ${eventData.action_url},
120
+ ${eventData.sitesearchkeyword},
121
+ ${eventData.action_title},
122
+ ${eventData.visitorid},
123
+ ${eventData.referrertype},
124
+ ${eventData.referrername},
125
+ ${eventData.resolution}
126
+ )
127
+ `.execute(db);
128
+ }
129
+ catch (err) {
130
+ // Add context for troubleshooting
131
+ const minimalContext = {
132
+ action_id: eventData.action_id,
133
+ action_timestamp: eventData.action_timestamp,
134
+ idsite: eventData.idsite,
135
+ idvisit: eventData.idvisit
136
+ };
137
+ console.error('importEvent(): failed to insert event', minimalContext);
138
+ // Log error details but avoid exposing sensitive information
139
+ console.error('importEvent(): error', err instanceof Error ? err.message : 'Unknown error');
140
+ throw err;
141
+ }
73
142
  });
74
- exports.importEvent = importEvent;
75
143
  const matomoProps = [
76
144
  'idSite',
77
145
  'idVisit',
@@ -105,11 +173,12 @@ const actionProps = {
105
173
  action_url: (action) => action.url,
106
174
  sitesearchkeyword: (action) => action.siteSearchKeyword
107
175
  };
108
- const getEventsFromMatomoVisit = (matomoVisit) => {
176
+ export const getEventsFromMatomoVisit = (matomoVisit) => {
109
177
  return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
178
+ var _a;
110
179
  const usercustomproperties = {};
111
180
  for (let k = 1; k < 10; k++) {
112
- const property = actionDetail.customVariables && actionDetail.customVariables[k];
181
+ const property = (_a = actionDetail.customVariables) === null || _a === void 0 ? void 0 : _a[k];
113
182
  if (!property)
114
183
  continue; // max 10 custom variables
115
184
  //@ts-expect-error implicit any type
@@ -140,4 +209,3 @@ const getEventsFromMatomoVisit = (matomoVisit) => {
140
209
  return event;
141
210
  });
142
211
  };
143
- exports.getEventsFromMatomoVisit = getEventsFromMatomoVisit;
package/dist/index.js CHANGED
@@ -1,4 +1,3 @@
1
- "use strict";
2
1
  var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
2
  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
3
  return new (P || (P = Promise))(function (resolve, reject) {
@@ -8,27 +7,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
8
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
9
8
  });
10
9
  };
11
- var __importDefault = (this && this.__importDefault) || function (mod) {
12
- return (mod && mod.__esModule) ? mod : { "default": mod };
13
- };
14
- Object.defineProperty(exports, "__esModule", { value: true });
15
- const eachDayOfInterval_1 = __importDefault(require("date-fns/eachDayOfInterval"));
16
- const debug_1 = __importDefault(require("debug"));
17
- const kysely_1 = require("kysely");
18
- const p_all_1 = __importDefault(require("p-all"));
19
- const config_1 = require("./config");
20
- const db_1 = require("./db");
21
- const importDate_1 = require("./importDate");
22
- const PiwikClient_1 = __importDefault(require("./PiwikClient"));
23
- const debug = (0, debug_1.default)('index');
10
+ import { eachDayOfInterval } from 'date-fns';
11
+ import startDebug from 'debug';
12
+ import { sql } from 'kysely';
13
+ import pAll from 'p-all';
14
+ import { DESTINATION_TABLE, INITIAL_OFFSET, MATOMO_KEY, MATOMO_SITE, MATOMO_URL } from './config.js';
15
+ import { db } from './db.js';
16
+ import { importDate } from './importDate.js';
17
+ import PiwikClient from './PiwikClient.js';
18
+ const debug = startDebug('index');
24
19
  function run(date) {
25
20
  return __awaiter(this, void 0, void 0, function* () {
26
21
  console.log(`🚀 Starting data import process`);
27
22
  debug('run, date=' + date);
28
23
  console.log(`🔗 Initializing Matomo client`);
29
- console.log(` - Matomo URL: ${config_1.MATOMO_URL}`);
30
- console.log(` - Matomo Site ID: ${config_1.MATOMO_SITE}`);
31
- const piwik = new PiwikClient_1.default(config_1.MATOMO_URL, config_1.MATOMO_KEY);
24
+ console.log(` - Matomo URL: ${MATOMO_URL}`);
25
+ console.log(` - Matomo Site ID: ${MATOMO_SITE}`);
26
+ const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
32
27
  console.log(`📅 Determining reference date for import...`);
33
28
  // priority:
34
29
  // - optional parameter date
@@ -40,13 +35,9 @@ function run(date) {
40
35
  referenceDate = new Date(date);
41
36
  console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
42
37
  }
43
- if (!referenceDate && process.env.STARTDATE) {
44
- referenceDate = new Date(process.env.STARTDATE);
45
- console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
46
- }
47
38
  if (!referenceDate) {
48
39
  console.log(`🔍 Looking for last event in database...`);
49
- referenceDate = yield findLastEventInMatomo(db_1.db);
40
+ referenceDate = yield findLastEventInMatomo(db);
50
41
  if (referenceDate) {
51
42
  console.log(`✅ Found last event, starting from: ${referenceDate.toISOString()}`);
52
43
  }
@@ -54,12 +45,16 @@ function run(date) {
54
45
  console.log(`ℹ️ No previous events found in database`);
55
46
  }
56
47
  }
48
+ if (!referenceDate && process.env.STARTDATE) {
49
+ referenceDate = new Date(process.env.STARTDATE);
50
+ console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
51
+ }
57
52
  if (!referenceDate) {
58
- referenceDate = new Date(new Date().getTime() - +config_1.INITIAL_OFFSET * 24 * 60 * 60 * 1000);
59
- console.log(`✅ Using default offset (${config_1.INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
53
+ referenceDate = new Date(new Date().getTime() - +INITIAL_OFFSET * 24 * 60 * 60 * 1000);
54
+ console.log(`✅ Using default offset (${INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
60
55
  }
61
56
  const endDate = new Date(new Date().getTime() + 24 * 60 * 60 * 1000);
62
- const dates = (0, eachDayOfInterval_1.default)({
57
+ const dates = eachDayOfInterval({
63
58
  start: referenceDate,
64
59
  end: endDate
65
60
  });
@@ -70,9 +65,9 @@ function run(date) {
70
65
  debug(`import starting at : ${dates[0].toISOString()}`);
71
66
  console.log(`🔄 Starting sequential import for each date...`);
72
67
  // for each date, serial-import data
73
- const res = yield (0, p_all_1.default)(dates.map((date, index) => () => {
68
+ const res = yield pAll(dates.map((date, index) => () => {
74
69
  console.log(`📅 Processing date ${index + 1}/${dates.length}: ${date.toISOString().split('T')[0]}`);
75
- return (0, importDate_1.importDate)(piwik.api.bind(piwik), date);
70
+ return importDate(piwik.api.bind(piwik), date);
76
71
  }), { concurrency: 1, stopOnError: true });
77
72
  const totalEvents = res.flat().length;
78
73
  console.log(`✅ Import process completed`);
@@ -83,26 +78,11 @@ function run(date) {
83
78
  return res;
84
79
  });
85
80
  }
86
- exports.default = run;
87
- if (require.main === module) {
88
- ;
89
- (() => __awaiter(void 0, void 0, void 0, function* () {
90
- if (!config_1.MATOMO_SITE)
91
- return console.error('Missing env MATOMO_SITE');
92
- if (!config_1.MATOMO_KEY)
93
- return console.error('Missing env MATOMO_KEY');
94
- if (!config_1.PGDATABASE)
95
- return console.error('Missing env PGDATABASE');
96
- yield run();
97
- debug('run finished');
98
- db_1.db.destroy();
99
- }))();
100
- }
101
81
  function findLastEventInMatomo(db) {
102
82
  return __awaiter(this, void 0, void 0, function* () {
103
83
  const latest = yield db
104
- .selectFrom(config_1.DESTINATION_TABLE)
105
- .select((0, kysely_1.sql) `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
84
+ .selectFrom(DESTINATION_TABLE)
85
+ .select(sql `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
106
86
  .orderBy('action_timestamp', 'desc')
107
87
  .limit(1)
108
88
  .executeTakeFirst();
@@ -114,3 +94,4 @@ function findLastEventInMatomo(db) {
114
94
  return null;
115
95
  });
116
96
  }
97
+ export default run;
@@ -1,27 +1,3 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
1
  var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
2
  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
3
  return new (P || (P = Promise))(function (resolve, reject) {
@@ -31,19 +7,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
31
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
32
8
  });
33
9
  };
34
- Object.defineProperty(exports, "__esModule", { value: true });
35
- const fs_1 = require("fs");
36
- const kysely_1 = require("kysely");
37
- const path = __importStar(require("path"));
38
- const db_1 = require("./db");
10
+ import { promises as fs } from 'fs';
11
+ import { FileMigrationProvider, Migrator } from 'kysely';
12
+ import * as path from 'path';
13
+ import { db } from './db.js';
39
14
  function migrateDown() {
40
15
  return __awaiter(this, void 0, void 0, function* () {
41
- const migrator = new kysely_1.Migrator({
42
- db: db_1.db,
43
- provider: new kysely_1.FileMigrationProvider({
44
- fs: fs_1.promises,
16
+ const migrator = new Migrator({
17
+ db,
18
+ provider: new FileMigrationProvider({
19
+ fs,
45
20
  path,
46
- migrationFolder: __dirname + '/migrations'
21
+ migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
47
22
  })
48
23
  });
49
24
  const { error, results } = yield migrator.migrateDown();
@@ -60,7 +35,7 @@ function migrateDown() {
60
35
  console.error(error);
61
36
  process.exit(1);
62
37
  }
63
- yield db_1.db.destroy();
38
+ yield db.destroy();
64
39
  });
65
40
  }
66
41
  migrateDown();
@@ -1,27 +1,3 @@
1
- "use strict";
2
- var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
- if (k2 === undefined) k2 = k;
4
- var desc = Object.getOwnPropertyDescriptor(m, k);
5
- if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
- desc = { enumerable: true, get: function() { return m[k]; } };
7
- }
8
- Object.defineProperty(o, k2, desc);
9
- }) : (function(o, m, k, k2) {
10
- if (k2 === undefined) k2 = k;
11
- o[k2] = m[k];
12
- }));
13
- var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
- Object.defineProperty(o, "default", { enumerable: true, value: v });
15
- }) : function(o, v) {
16
- o["default"] = v;
17
- });
18
- var __importStar = (this && this.__importStar) || function (mod) {
19
- if (mod && mod.__esModule) return mod;
20
- var result = {};
21
- if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
- __setModuleDefault(result, mod);
23
- return result;
24
- };
25
1
  var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
2
  function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
3
  return new (P || (P = Promise))(function (resolve, reject) {
@@ -31,53 +7,61 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
31
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
32
8
  });
33
9
  };
34
- Object.defineProperty(exports, "__esModule", { value: true });
35
- const fs_1 = require("fs");
36
- const kysely_1 = require("kysely");
37
- const path = __importStar(require("path"));
38
- const config_1 = require("./config");
39
- const db_1 = require("./db");
10
+ import { promises as fs } from 'fs';
11
+ import { FileMigrationProvider, Migrator } from 'kysely';
12
+ import * as path from 'path';
13
+ import { MATOMO_TABLE_NAME } from './config.js';
14
+ import { db } from './db.js';
40
15
  function migrateToLatest() {
41
16
  return __awaiter(this, void 0, void 0, function* () {
42
- const migrator = new kysely_1.Migrator({
43
- db: db_1.db,
44
- provider: new kysely_1.FileMigrationProvider({
45
- fs: fs_1.promises,
46
- path,
47
- migrationFolder: __dirname + '/migrations'
48
- }),
49
- // allow to have mutliple migratable instances in a single schema
50
- migrationTableName: `${config_1.MATOMO_TABLE_NAME}_migration`,
51
- migrationLockTableName: `${config_1.MATOMO_TABLE_NAME}_migration_lock`
52
- });
53
- const { error, results } = yield migrator.migrateToLatest();
54
- results === null || results === void 0 ? void 0 : results.forEach((it) => {
55
- if (it.status === 'Success') {
56
- console.log(`migration "${it.migrationName}" was executed successfully`);
57
- }
58
- else if (it.status === 'Error') {
59
- console.error(`failed to execute migration "${it.migrationName}"`);
17
+ console.log(`Starting migrate to latest`);
18
+ try {
19
+ const migrator = new Migrator({
20
+ db,
21
+ provider: new FileMigrationProvider({
22
+ fs,
23
+ path,
24
+ migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
25
+ }),
26
+ // allow to have mutliple migratable instances in a single schema
27
+ migrationTableName: `${MATOMO_TABLE_NAME}_migration`,
28
+ migrationLockTableName: `${MATOMO_TABLE_NAME}_migration_lock`
29
+ });
30
+ const { error, results } = yield migrator.migrateToLatest();
31
+ results === null || results === void 0 ? void 0 : results.forEach((it) => {
32
+ if (it.status === 'Success') {
33
+ console.log(`migration "${it.migrationName}" was executed successfully`);
34
+ }
35
+ else if (it.status === 'Error') {
36
+ console.error(`failed to execute migration "${it.migrationName}"`);
37
+ }
38
+ });
39
+ if (error) {
40
+ console.error('failed to migrate');
41
+ console.error(error);
42
+ process.exit(1);
60
43
  }
61
- });
62
- if (error) {
63
- console.error('failed to migrate');
64
- console.error(error);
65
- process.exit(1);
66
- }
67
- else {
68
- if (!(results === null || results === void 0 ? void 0 : results.length)) {
44
+ else if (!(results === null || results === void 0 ? void 0 : results.length)) {
69
45
  console.log('No migration to run');
70
46
  }
71
47
  }
48
+ catch (uncaughtError) {
49
+ console.error('UNCAUGHT ERROR during migration:');
50
+ console.error('Error message:', uncaughtError instanceof Error
51
+ ? uncaughtError.message
52
+ : String(uncaughtError));
53
+ console.error('Error stack:', uncaughtError instanceof Error
54
+ ? uncaughtError.stack
55
+ : 'No stack trace available');
56
+ console.error('Full error object:', uncaughtError);
57
+ process.exit(1);
58
+ }
72
59
  });
73
60
  }
74
- exports.default = migrateToLatest;
75
- function start() {
61
+ export default migrateToLatest;
62
+ export function startMigration() {
76
63
  return __awaiter(this, void 0, void 0, function* () {
77
64
  yield migrateToLatest();
78
- yield db_1.db.destroy();
65
+ // Don't destroy the db connection here since the main application will need it
79
66
  });
80
67
  }
81
- if (require.main === module) {
82
- start();
83
- }