@socialgouv/matomo-postgres 2.2.0 → 2.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +199 -32
- package/bin/index.js +10 -12
- package/dist/PiwikClient.js +5 -31
- package/dist/__tests__/importDate.test.js +49 -44
- package/dist/__tests__/importEvent.test.js +3 -8
- package/dist/__tests__/run.test.js +22 -19
- package/dist/config.js +11 -11
- package/dist/db.js +10 -15
- package/dist/importDate.js +19 -27
- package/dist/importEvent.js +127 -59
- package/dist/index.js +25 -44
- package/dist/migrate-down.js +10 -35
- package/dist/migrate-latest.js +46 -62
- package/dist/migrations/20230301-01-initial.js +4 -9
- package/dist/migrations/20230301-02-indexes.js +4 -9
- package/dist/migrations/20250425-01-add-resolution.js +18 -11
- package/dist/migrations/20250715-01-weekly-partitioning.js +28 -31
- package/dist/migrations/20250908-01-convention-analysis-index.js +29 -0
- package/package.json +6 -6
package/dist/importDate.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,36 +7,31 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const p_all_1 = __importDefault(require("p-all"));
|
|
20
|
-
const config_1 = require("./config");
|
|
21
|
-
const db_1 = require("./db");
|
|
22
|
-
const importEvent_1 = require("./importEvent");
|
|
23
|
-
const debug = (0, debug_1.default)('importDate');
|
|
10
|
+
import { formatISO } from 'date-fns';
|
|
11
|
+
import startDebug from 'debug';
|
|
12
|
+
import { sql } from 'kysely';
|
|
13
|
+
import pAll from 'p-all';
|
|
14
|
+
import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
|
|
15
|
+
import { db } from './db.js';
|
|
16
|
+
import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
|
|
17
|
+
const debug = startDebug('importDate');
|
|
24
18
|
/** return date as ISO yyyy-mm-dd */
|
|
25
|
-
const isoDate = (date) => (
|
|
19
|
+
const isoDate = (date) => formatISO(date, { representation: 'date' });
|
|
26
20
|
/** check how many visits complete for a given date */
|
|
27
21
|
const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
|
|
28
|
-
const result = yield
|
|
29
|
-
.selectFrom(
|
|
30
|
-
.select(
|
|
22
|
+
const result = yield db
|
|
23
|
+
.selectFrom(DESTINATION_TABLE)
|
|
24
|
+
.select(db.fn.count('idvisit').distinct().as('count'))
|
|
31
25
|
// UTC to be iso with matomo matomo data
|
|
32
|
-
.where(
|
|
26
|
+
.where(sql `date(timezone('UTC', action_timestamp))`, '=', date)
|
|
33
27
|
.executeTakeFirst();
|
|
34
28
|
// start at previous visit in case action didnt finished to record
|
|
35
29
|
const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
|
|
36
30
|
return count;
|
|
37
31
|
});
|
|
38
32
|
/** import all event from givent date */
|
|
39
|
-
const importDate = (
|
|
40
|
-
const limit = parseInt(
|
|
33
|
+
export const importDate = (piwikApi_1, date_1, ...args_1) => __awaiter(void 0, [piwikApi_1, date_1, ...args_1], void 0, function* (piwikApi, date, filterOffset = 0) {
|
|
34
|
+
const limit = parseInt(RESULTPERPAGE);
|
|
41
35
|
const offset = filterOffset || (yield getRecordsCount(isoDate(date)));
|
|
42
36
|
if (!offset) {
|
|
43
37
|
debug(`${isoDate(date)}: load ${limit} visits`);
|
|
@@ -54,7 +48,7 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
|
|
|
54
48
|
filter_limit: limit,
|
|
55
49
|
filter_offset: offset,
|
|
56
50
|
filter_sort_order: 'asc',
|
|
57
|
-
idSite:
|
|
51
|
+
idSite: MATOMO_SITE
|
|
58
52
|
}, (err, visits = []) => {
|
|
59
53
|
if (err) {
|
|
60
54
|
console.error('err', err);
|
|
@@ -64,7 +58,7 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
|
|
|
64
58
|
}));
|
|
65
59
|
debug(`fetched ${visits.length} visits`);
|
|
66
60
|
// flatten all events
|
|
67
|
-
const eventsFromVisits = visits.flatMap(
|
|
61
|
+
const eventsFromVisits = visits.flatMap(getEventsFromMatomoVisit);
|
|
68
62
|
const allEvents = eventsFromVisits.filter((_event) => {
|
|
69
63
|
return true;
|
|
70
64
|
});
|
|
@@ -74,15 +68,13 @@ const importDate = (piwikApi, date, filterOffset = 0) => __awaiter(void 0, void
|
|
|
74
68
|
}
|
|
75
69
|
debug(`import ${allEvents.length} events`);
|
|
76
70
|
// serial-import events into PG
|
|
77
|
-
yield (
|
|
71
|
+
yield pAll(allEvents.map((event) => () => importEvent(event)), { concurrency: 10, stopOnError: true });
|
|
78
72
|
// continue to next page if necessary
|
|
79
73
|
if (visits.length === limit) {
|
|
80
74
|
const nextOffset = offset + limit;
|
|
81
|
-
const nextEvents = yield
|
|
75
|
+
const nextEvents = yield importDate(piwikApi, date, nextOffset);
|
|
82
76
|
return [...allEvents, ...(nextEvents || [])];
|
|
83
77
|
}
|
|
84
78
|
debug(`finished importing ${isoDate(date)}, offset ${offset}`);
|
|
85
79
|
return allEvents;
|
|
86
80
|
});
|
|
87
|
-
exports.importDate = importDate;
|
|
88
|
-
module.exports = { importDate: exports.importDate };
|
package/dist/importEvent.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,10 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const kysely_1 = require("kysely");
|
|
14
|
-
const db_1 = require("./db");
|
|
10
|
+
import { sql } from 'kysely';
|
|
11
|
+
import { db } from './db.js';
|
|
15
12
|
/**
|
|
16
13
|
*
|
|
17
14
|
* @param {Client} client
|
|
@@ -19,59 +16,130 @@ const db_1 = require("./db");
|
|
|
19
16
|
*
|
|
20
17
|
* @return {Promise<void>}
|
|
21
18
|
*/
|
|
22
|
-
const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
|
|
19
|
+
export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
20
|
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
|
|
24
|
-
//
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
21
|
+
// Build a sanitized, typed data object to reduce drift and ensure defaults in one place
|
|
22
|
+
const eventData = {
|
|
23
|
+
action_id: (_a = event.action_id) !== null && _a !== void 0 ? _a : '',
|
|
24
|
+
action_timestamp: event.action_timestamp
|
|
25
|
+
? new Date(event.action_timestamp)
|
|
26
|
+
: new Date(),
|
|
27
|
+
idsite: (_b = event.idsite) !== null && _b !== void 0 ? _b : '',
|
|
28
|
+
idvisit: (_c = event.idvisit) !== null && _c !== void 0 ? _c : '',
|
|
29
|
+
actions: (_d = event.actions) !== null && _d !== void 0 ? _d : null,
|
|
30
|
+
country: (_e = event.country) !== null && _e !== void 0 ? _e : null,
|
|
31
|
+
region: (_f = event.region) !== null && _f !== void 0 ? _f : null,
|
|
32
|
+
city: (_g = event.city) !== null && _g !== void 0 ? _g : null,
|
|
33
|
+
operatingsystemname: (_h = event.operatingsystemname) !== null && _h !== void 0 ? _h : null,
|
|
34
|
+
devicemodel: (_j = event.devicemodel) !== null && _j !== void 0 ? _j : null,
|
|
35
|
+
devicebrand: (_k = event.devicebrand) !== null && _k !== void 0 ? _k : null,
|
|
36
|
+
visitduration: (_l = event.visitduration) !== null && _l !== void 0 ? _l : null,
|
|
37
|
+
dayssincefirstvisit: (_m = event.dayssincefirstvisit) !== null && _m !== void 0 ? _m : null,
|
|
38
|
+
visitortype: (_o = event.visitortype) !== null && _o !== void 0 ? _o : null,
|
|
39
|
+
sitename: (_p = event.sitename) !== null && _p !== void 0 ? _p : null,
|
|
40
|
+
userid: (_q = event.userid) !== null && _q !== void 0 ? _q : null,
|
|
41
|
+
serverdateprettyfirstaction: event.serverdateprettyfirstaction
|
|
42
|
+
? new Date(event.serverdateprettyfirstaction)
|
|
43
|
+
: null,
|
|
44
|
+
action_type: (_r = event.action_type) !== null && _r !== void 0 ? _r : '',
|
|
45
|
+
action_eventcategory: (_s = event.action_eventcategory) !== null && _s !== void 0 ? _s : '',
|
|
46
|
+
action_eventaction: (_t = event.action_eventaction) !== null && _t !== void 0 ? _t : '',
|
|
47
|
+
action_eventname: (_u = event.action_eventname) !== null && _u !== void 0 ? _u : '',
|
|
48
|
+
action_eventvalue: event.action_eventvalue
|
|
49
|
+
? Number(event.action_eventvalue)
|
|
50
|
+
: 0,
|
|
51
|
+
action_timespent: (_v = event.action_timespent) !== null && _v !== void 0 ? _v : '0',
|
|
52
|
+
usercustomproperties: (_w = event.usercustomproperties) !== null && _w !== void 0 ? _w : null,
|
|
53
|
+
usercustomdimensions: (_x = event.usercustomdimensions) !== null && _x !== void 0 ? _x : null,
|
|
54
|
+
dimension1: (_y = event.dimension1) !== null && _y !== void 0 ? _y : null,
|
|
55
|
+
dimension2: (_z = event.dimension2) !== null && _z !== void 0 ? _z : null,
|
|
56
|
+
dimension3: (_0 = event.dimension3) !== null && _0 !== void 0 ? _0 : null,
|
|
57
|
+
dimension4: (_1 = event.dimension4) !== null && _1 !== void 0 ? _1 : null,
|
|
58
|
+
dimension5: (_2 = event.dimension5) !== null && _2 !== void 0 ? _2 : null,
|
|
59
|
+
dimension6: (_3 = event.dimension6) !== null && _3 !== void 0 ? _3 : null,
|
|
60
|
+
dimension7: (_4 = event.dimension7) !== null && _4 !== void 0 ? _4 : null,
|
|
61
|
+
dimension8: (_5 = event.dimension8) !== null && _5 !== void 0 ? _5 : null,
|
|
62
|
+
dimension9: (_6 = event.dimension9) !== null && _6 !== void 0 ? _6 : null,
|
|
63
|
+
dimension10: (_7 = event.dimension10) !== null && _7 !== void 0 ? _7 : null,
|
|
64
|
+
action_url: (_8 = event.action_url) !== null && _8 !== void 0 ? _8 : null,
|
|
65
|
+
sitesearchkeyword: (_9 = event.sitesearchkeyword) !== null && _9 !== void 0 ? _9 : null,
|
|
66
|
+
action_title: (_10 = event.action_title) !== null && _10 !== void 0 ? _10 : null,
|
|
67
|
+
visitorid: (_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null,
|
|
68
|
+
referrertype: (_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null,
|
|
69
|
+
referrername: (_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null,
|
|
70
|
+
resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null
|
|
71
|
+
};
|
|
72
|
+
// Minimal runtime validation for required fields
|
|
73
|
+
if (!eventData.action_id || eventData.action_id.trim().length === 0) {
|
|
74
|
+
throw new Error('importEvent(): action_id is required and cannot be empty');
|
|
75
|
+
}
|
|
76
|
+
if (!(eventData.action_timestamp instanceof Date) ||
|
|
77
|
+
isNaN(eventData.action_timestamp.getTime())) {
|
|
78
|
+
throw new Error('importEvent(): action_timestamp is invalid');
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
// Keep the stored procedure but centralize mapping to avoid parameter mis-ordering
|
|
82
|
+
yield sql `
|
|
83
|
+
SELECT insert_into_matomo_partitioned(
|
|
84
|
+
${eventData.action_id},
|
|
85
|
+
${eventData.action_timestamp},
|
|
86
|
+
${eventData.idsite},
|
|
87
|
+
${eventData.idvisit},
|
|
88
|
+
${eventData.actions},
|
|
89
|
+
${eventData.country},
|
|
90
|
+
${eventData.region},
|
|
91
|
+
${eventData.city},
|
|
92
|
+
${eventData.operatingsystemname},
|
|
93
|
+
${eventData.devicemodel},
|
|
94
|
+
${eventData.devicebrand},
|
|
95
|
+
${eventData.visitduration},
|
|
96
|
+
${eventData.dayssincefirstvisit},
|
|
97
|
+
${eventData.visitortype},
|
|
98
|
+
${eventData.sitename},
|
|
99
|
+
${eventData.userid},
|
|
100
|
+
${eventData.serverdateprettyfirstaction},
|
|
101
|
+
${eventData.action_type},
|
|
102
|
+
${eventData.action_eventcategory},
|
|
103
|
+
${eventData.action_eventaction},
|
|
104
|
+
${eventData.action_eventname},
|
|
105
|
+
${eventData.action_eventvalue},
|
|
106
|
+
${eventData.action_timespent},
|
|
107
|
+
${eventData.usercustomproperties},
|
|
108
|
+
${eventData.usercustomdimensions},
|
|
109
|
+
${eventData.dimension1},
|
|
110
|
+
${eventData.dimension2},
|
|
111
|
+
${eventData.dimension3},
|
|
112
|
+
${eventData.dimension4},
|
|
113
|
+
${eventData.dimension5},
|
|
114
|
+
${eventData.dimension6},
|
|
115
|
+
${eventData.dimension7},
|
|
116
|
+
${eventData.dimension8},
|
|
117
|
+
${eventData.dimension9},
|
|
118
|
+
${eventData.dimension10},
|
|
119
|
+
${eventData.action_url},
|
|
120
|
+
${eventData.sitesearchkeyword},
|
|
121
|
+
${eventData.action_title},
|
|
122
|
+
${eventData.visitorid},
|
|
123
|
+
${eventData.referrertype},
|
|
124
|
+
${eventData.referrername},
|
|
125
|
+
${eventData.resolution}
|
|
126
|
+
)
|
|
127
|
+
`.execute(db);
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
// Add context for troubleshooting
|
|
131
|
+
const minimalContext = {
|
|
132
|
+
action_id: eventData.action_id,
|
|
133
|
+
action_timestamp: eventData.action_timestamp,
|
|
134
|
+
idsite: eventData.idsite,
|
|
135
|
+
idvisit: eventData.idvisit
|
|
136
|
+
};
|
|
137
|
+
console.error('importEvent(): failed to insert event', minimalContext);
|
|
138
|
+
// Log error details but avoid exposing sensitive information
|
|
139
|
+
console.error('importEvent(): error', err instanceof Error ? err.message : 'Unknown error');
|
|
140
|
+
throw err;
|
|
141
|
+
}
|
|
73
142
|
});
|
|
74
|
-
exports.importEvent = importEvent;
|
|
75
143
|
const matomoProps = [
|
|
76
144
|
'idSite',
|
|
77
145
|
'idVisit',
|
|
@@ -105,11 +173,12 @@ const actionProps = {
|
|
|
105
173
|
action_url: (action) => action.url,
|
|
106
174
|
sitesearchkeyword: (action) => action.siteSearchKeyword
|
|
107
175
|
};
|
|
108
|
-
const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
176
|
+
export const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
109
177
|
return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
|
|
178
|
+
var _a;
|
|
110
179
|
const usercustomproperties = {};
|
|
111
180
|
for (let k = 1; k < 10; k++) {
|
|
112
|
-
const property = actionDetail.customVariables
|
|
181
|
+
const property = (_a = actionDetail.customVariables) === null || _a === void 0 ? void 0 : _a[k];
|
|
113
182
|
if (!property)
|
|
114
183
|
continue; // max 10 custom variables
|
|
115
184
|
//@ts-expect-error implicit any type
|
|
@@ -140,4 +209,3 @@ const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
|
140
209
|
return event;
|
|
141
210
|
});
|
|
142
211
|
};
|
|
143
|
-
exports.getEventsFromMatomoVisit = getEventsFromMatomoVisit;
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,27 +7,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const db_1 = require("./db");
|
|
21
|
-
const importDate_1 = require("./importDate");
|
|
22
|
-
const PiwikClient_1 = __importDefault(require("./PiwikClient"));
|
|
23
|
-
const debug = (0, debug_1.default)('index');
|
|
10
|
+
import { eachDayOfInterval } from 'date-fns';
|
|
11
|
+
import startDebug from 'debug';
|
|
12
|
+
import { sql } from 'kysely';
|
|
13
|
+
import pAll from 'p-all';
|
|
14
|
+
import { DESTINATION_TABLE, INITIAL_OFFSET, MATOMO_KEY, MATOMO_SITE, MATOMO_URL } from './config.js';
|
|
15
|
+
import { db } from './db.js';
|
|
16
|
+
import { importDate } from './importDate.js';
|
|
17
|
+
import PiwikClient from './PiwikClient.js';
|
|
18
|
+
const debug = startDebug('index');
|
|
24
19
|
function run(date) {
|
|
25
20
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
21
|
console.log(`🚀 Starting data import process`);
|
|
27
22
|
debug('run, date=' + date);
|
|
28
23
|
console.log(`🔗 Initializing Matomo client`);
|
|
29
|
-
console.log(` - Matomo URL: ${
|
|
30
|
-
console.log(` - Matomo Site ID: ${
|
|
31
|
-
const piwik = new
|
|
24
|
+
console.log(` - Matomo URL: ${MATOMO_URL}`);
|
|
25
|
+
console.log(` - Matomo Site ID: ${MATOMO_SITE}`);
|
|
26
|
+
const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
|
|
32
27
|
console.log(`📅 Determining reference date for import...`);
|
|
33
28
|
// priority:
|
|
34
29
|
// - optional parameter date
|
|
@@ -40,13 +35,9 @@ function run(date) {
|
|
|
40
35
|
referenceDate = new Date(date);
|
|
41
36
|
console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
|
|
42
37
|
}
|
|
43
|
-
if (!referenceDate && process.env.STARTDATE) {
|
|
44
|
-
referenceDate = new Date(process.env.STARTDATE);
|
|
45
|
-
console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
|
|
46
|
-
}
|
|
47
38
|
if (!referenceDate) {
|
|
48
39
|
console.log(`🔍 Looking for last event in database...`);
|
|
49
|
-
referenceDate = yield findLastEventInMatomo(
|
|
40
|
+
referenceDate = yield findLastEventInMatomo(db);
|
|
50
41
|
if (referenceDate) {
|
|
51
42
|
console.log(`✅ Found last event, starting from: ${referenceDate.toISOString()}`);
|
|
52
43
|
}
|
|
@@ -54,12 +45,16 @@ function run(date) {
|
|
|
54
45
|
console.log(`ℹ️ No previous events found in database`);
|
|
55
46
|
}
|
|
56
47
|
}
|
|
48
|
+
if (!referenceDate && process.env.STARTDATE) {
|
|
49
|
+
referenceDate = new Date(process.env.STARTDATE);
|
|
50
|
+
console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
|
|
51
|
+
}
|
|
57
52
|
if (!referenceDate) {
|
|
58
|
-
referenceDate = new Date(new Date().getTime() - +
|
|
59
|
-
console.log(`✅ Using default offset (${
|
|
53
|
+
referenceDate = new Date(new Date().getTime() - +INITIAL_OFFSET * 24 * 60 * 60 * 1000);
|
|
54
|
+
console.log(`✅ Using default offset (${INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
|
|
60
55
|
}
|
|
61
56
|
const endDate = new Date(new Date().getTime() + 24 * 60 * 60 * 1000);
|
|
62
|
-
const dates = (
|
|
57
|
+
const dates = eachDayOfInterval({
|
|
63
58
|
start: referenceDate,
|
|
64
59
|
end: endDate
|
|
65
60
|
});
|
|
@@ -70,9 +65,9 @@ function run(date) {
|
|
|
70
65
|
debug(`import starting at : ${dates[0].toISOString()}`);
|
|
71
66
|
console.log(`🔄 Starting sequential import for each date...`);
|
|
72
67
|
// for each date, serial-import data
|
|
73
|
-
const res = yield (
|
|
68
|
+
const res = yield pAll(dates.map((date, index) => () => {
|
|
74
69
|
console.log(`📅 Processing date ${index + 1}/${dates.length}: ${date.toISOString().split('T')[0]}`);
|
|
75
|
-
return
|
|
70
|
+
return importDate(piwik.api.bind(piwik), date);
|
|
76
71
|
}), { concurrency: 1, stopOnError: true });
|
|
77
72
|
const totalEvents = res.flat().length;
|
|
78
73
|
console.log(`✅ Import process completed`);
|
|
@@ -83,26 +78,11 @@ function run(date) {
|
|
|
83
78
|
return res;
|
|
84
79
|
});
|
|
85
80
|
}
|
|
86
|
-
exports.default = run;
|
|
87
|
-
if (require.main === module) {
|
|
88
|
-
;
|
|
89
|
-
(() => __awaiter(void 0, void 0, void 0, function* () {
|
|
90
|
-
if (!config_1.MATOMO_SITE)
|
|
91
|
-
return console.error('Missing env MATOMO_SITE');
|
|
92
|
-
if (!config_1.MATOMO_KEY)
|
|
93
|
-
return console.error('Missing env MATOMO_KEY');
|
|
94
|
-
if (!config_1.PGDATABASE)
|
|
95
|
-
return console.error('Missing env PGDATABASE');
|
|
96
|
-
yield run();
|
|
97
|
-
debug('run finished');
|
|
98
|
-
db_1.db.destroy();
|
|
99
|
-
}))();
|
|
100
|
-
}
|
|
101
81
|
function findLastEventInMatomo(db) {
|
|
102
82
|
return __awaiter(this, void 0, void 0, function* () {
|
|
103
83
|
const latest = yield db
|
|
104
|
-
.selectFrom(
|
|
105
|
-
.select(
|
|
84
|
+
.selectFrom(DESTINATION_TABLE)
|
|
85
|
+
.select(sql `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
|
|
106
86
|
.orderBy('action_timestamp', 'desc')
|
|
107
87
|
.limit(1)
|
|
108
88
|
.executeTakeFirst();
|
|
@@ -114,3 +94,4 @@ function findLastEventInMatomo(db) {
|
|
|
114
94
|
return null;
|
|
115
95
|
});
|
|
116
96
|
}
|
|
97
|
+
export default run;
|
package/dist/migrate-down.js
CHANGED
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -31,19 +7,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
31
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
8
|
});
|
|
33
9
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const db_1 = require("./db");
|
|
10
|
+
import { promises as fs } from 'fs';
|
|
11
|
+
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
import { db } from './db.js';
|
|
39
14
|
function migrateDown() {
|
|
40
15
|
return __awaiter(this, void 0, void 0, function* () {
|
|
41
|
-
const migrator = new
|
|
42
|
-
db
|
|
43
|
-
provider: new
|
|
44
|
-
fs
|
|
16
|
+
const migrator = new Migrator({
|
|
17
|
+
db,
|
|
18
|
+
provider: new FileMigrationProvider({
|
|
19
|
+
fs,
|
|
45
20
|
path,
|
|
46
|
-
migrationFolder:
|
|
21
|
+
migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
|
|
47
22
|
})
|
|
48
23
|
});
|
|
49
24
|
const { error, results } = yield migrator.migrateDown();
|
|
@@ -60,7 +35,7 @@ function migrateDown() {
|
|
|
60
35
|
console.error(error);
|
|
61
36
|
process.exit(1);
|
|
62
37
|
}
|
|
63
|
-
yield
|
|
38
|
+
yield db.destroy();
|
|
64
39
|
});
|
|
65
40
|
}
|
|
66
41
|
migrateDown();
|
package/dist/migrate-latest.js
CHANGED
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -31,53 +7,61 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
31
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
8
|
});
|
|
33
9
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
const db_1 = require("./db");
|
|
10
|
+
import { promises as fs } from 'fs';
|
|
11
|
+
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
import { MATOMO_TABLE_NAME } from './config.js';
|
|
14
|
+
import { db } from './db.js';
|
|
40
15
|
function migrateToLatest() {
|
|
41
16
|
return __awaiter(this, void 0, void 0, function* () {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
17
|
+
console.log(`Starting migrate to latest`);
|
|
18
|
+
try {
|
|
19
|
+
const migrator = new Migrator({
|
|
20
|
+
db,
|
|
21
|
+
provider: new FileMigrationProvider({
|
|
22
|
+
fs,
|
|
23
|
+
path,
|
|
24
|
+
migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
|
|
25
|
+
}),
|
|
26
|
+
// allow to have mutliple migratable instances in a single schema
|
|
27
|
+
migrationTableName: `${MATOMO_TABLE_NAME}_migration`,
|
|
28
|
+
migrationLockTableName: `${MATOMO_TABLE_NAME}_migration_lock`
|
|
29
|
+
});
|
|
30
|
+
const { error, results } = yield migrator.migrateToLatest();
|
|
31
|
+
results === null || results === void 0 ? void 0 : results.forEach((it) => {
|
|
32
|
+
if (it.status === 'Success') {
|
|
33
|
+
console.log(`migration "${it.migrationName}" was executed successfully`);
|
|
34
|
+
}
|
|
35
|
+
else if (it.status === 'Error') {
|
|
36
|
+
console.error(`failed to execute migration "${it.migrationName}"`);
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
if (error) {
|
|
40
|
+
console.error('failed to migrate');
|
|
41
|
+
console.error(error);
|
|
42
|
+
process.exit(1);
|
|
60
43
|
}
|
|
61
|
-
|
|
62
|
-
if (error) {
|
|
63
|
-
console.error('failed to migrate');
|
|
64
|
-
console.error(error);
|
|
65
|
-
process.exit(1);
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
if (!(results === null || results === void 0 ? void 0 : results.length)) {
|
|
44
|
+
else if (!(results === null || results === void 0 ? void 0 : results.length)) {
|
|
69
45
|
console.log('No migration to run');
|
|
70
46
|
}
|
|
71
47
|
}
|
|
48
|
+
catch (uncaughtError) {
|
|
49
|
+
console.error('UNCAUGHT ERROR during migration:');
|
|
50
|
+
console.error('Error message:', uncaughtError instanceof Error
|
|
51
|
+
? uncaughtError.message
|
|
52
|
+
: String(uncaughtError));
|
|
53
|
+
console.error('Error stack:', uncaughtError instanceof Error
|
|
54
|
+
? uncaughtError.stack
|
|
55
|
+
: 'No stack trace available');
|
|
56
|
+
console.error('Full error object:', uncaughtError);
|
|
57
|
+
process.exit(1);
|
|
58
|
+
}
|
|
72
59
|
});
|
|
73
60
|
}
|
|
74
|
-
|
|
75
|
-
function
|
|
61
|
+
export default migrateToLatest;
|
|
62
|
+
export function startMigration() {
|
|
76
63
|
return __awaiter(this, void 0, void 0, function* () {
|
|
77
64
|
yield migrateToLatest();
|
|
78
|
-
|
|
65
|
+
// Don't destroy the db connection here since the main application will need it
|
|
79
66
|
});
|
|
80
67
|
}
|
|
81
|
-
if (require.main === module) {
|
|
82
|
-
start();
|
|
83
|
-
}
|