@socialgouv/matomo-postgres 2.3.3 → 2.3.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/package.json +10 -4
- package/dist/PiwikClient.js +0 -95
- package/dist/__tests__/importDate.test.js +0 -123
- package/dist/__tests__/importEvent.test.js +0 -9
- package/dist/__tests__/migration-imports.test.js +0 -38
- package/dist/__tests__/run.test.js +0 -91
- package/dist/__tests__/visit.json +0 -104
- package/dist/config.js +0 -12
- package/dist/db.js +0 -31
- package/dist/importDate.js +0 -83
- package/dist/importEvent.js +0 -214
- package/dist/index.js +0 -106
- package/dist/migrate-down.js +0 -41
- package/dist/migrate-latest.js +0 -67
- package/dist/migrations/20230301-01-initial.js +0 -65
- package/dist/migrations/20230301-02-indexes.js +0 -100
- package/dist/migrations/20250425-01-add-resolution.js +0 -38
- package/dist/migrations/20250715-01-weekly-partitioning.js +0 -359
- package/dist/migrations/20250908-01-convention-analysis-index.js +0 -29
package/dist/importDate.js
DELETED
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { formatISO } from 'date-fns';
|
|
11
|
-
import startDebug from 'debug';
|
|
12
|
-
import { sql } from 'kysely';
|
|
13
|
-
import pAll from 'p-all';
|
|
14
|
-
import { DESTINATION_TABLE, MATOMO_SITE, RESULTPERPAGE } from './config.js';
|
|
15
|
-
import { db, pool } from './db.js';
|
|
16
|
-
import { getEventsFromMatomoVisit, importEvent } from './importEvent.js';
|
|
17
|
-
const debug = startDebug('importDate');
|
|
18
|
-
/** return date as ISO yyyy-mm-dd */
|
|
19
|
-
const isoDate = (date) => formatISO(date, { representation: 'date' });
|
|
20
|
-
/** check how many visits complete for a given date */
|
|
21
|
-
const getRecordsCount = (date) => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
|
-
if (!pool || typeof pool.connect !== 'function') {
|
|
23
|
-
throw new Error('Database connection pool is invalid or undefined in getRecordsCount');
|
|
24
|
-
}
|
|
25
|
-
const result = yield db
|
|
26
|
-
.selectFrom(DESTINATION_TABLE)
|
|
27
|
-
.select(db.fn.count('idvisit').distinct().as('count'))
|
|
28
|
-
// UTC to be iso with matomo matomo data
|
|
29
|
-
.where(sql `date(timezone('UTC', action_timestamp))`, '=', date)
|
|
30
|
-
.executeTakeFirst();
|
|
31
|
-
// start at previous visit in case action didnt finished to record
|
|
32
|
-
const count = Math.max(0, (result && parseInt(result.count) - 1) || 0);
|
|
33
|
-
return count;
|
|
34
|
-
});
|
|
35
|
-
/** import all event from givent date */
|
|
36
|
-
export const importDate = (piwikApi_1, date_1, ...args_1) => __awaiter(void 0, [piwikApi_1, date_1, ...args_1], void 0, function* (piwikApi, date, filterOffset = 0) {
|
|
37
|
-
const limit = parseInt(RESULTPERPAGE);
|
|
38
|
-
const offset = filterOffset || (yield getRecordsCount(isoDate(date)));
|
|
39
|
-
if (!offset) {
|
|
40
|
-
debug(`${isoDate(date)}: load ${limit} visits`);
|
|
41
|
-
}
|
|
42
|
-
else {
|
|
43
|
-
debug(`${isoDate(date)}: load ${limit} more visits after ${offset}`);
|
|
44
|
-
}
|
|
45
|
-
// fetch visits details
|
|
46
|
-
const visits = yield new Promise((resolve) => piwikApi({
|
|
47
|
-
method: 'Live.getLastVisitsDetails',
|
|
48
|
-
period: 'day',
|
|
49
|
-
date: isoDate(date),
|
|
50
|
-
// minTimestamp: isoDate(new Date()) === isoDate(date) ? date.getTime() / 1000 : undefined, // if today, dont go further (??)
|
|
51
|
-
filter_limit: limit,
|
|
52
|
-
filter_offset: offset,
|
|
53
|
-
filter_sort_order: 'asc',
|
|
54
|
-
idSite: MATOMO_SITE
|
|
55
|
-
}, (err, visits = []) => {
|
|
56
|
-
if (err) {
|
|
57
|
-
console.error('err', err);
|
|
58
|
-
resolve([]);
|
|
59
|
-
}
|
|
60
|
-
return resolve(visits);
|
|
61
|
-
}));
|
|
62
|
-
debug(`fetched ${visits.length} visits`);
|
|
63
|
-
// flatten all events
|
|
64
|
-
const eventsFromVisits = visits.flatMap(getEventsFromMatomoVisit);
|
|
65
|
-
const allEvents = eventsFromVisits.filter((_event) => {
|
|
66
|
-
return true;
|
|
67
|
-
});
|
|
68
|
-
if (!allEvents.length) {
|
|
69
|
-
debug(`no more valid events after ${isoDate(date)}`);
|
|
70
|
-
return [];
|
|
71
|
-
}
|
|
72
|
-
debug(`import ${allEvents.length} events`);
|
|
73
|
-
// serial-import events into PG
|
|
74
|
-
yield pAll(allEvents.map((event) => () => importEvent(event)), { concurrency: 10, stopOnError: true });
|
|
75
|
-
// continue to next page if necessary
|
|
76
|
-
if (visits.length === limit) {
|
|
77
|
-
const nextOffset = offset + limit;
|
|
78
|
-
const nextEvents = yield importDate(piwikApi, date, nextOffset);
|
|
79
|
-
return [...allEvents, ...(nextEvents || [])];
|
|
80
|
-
}
|
|
81
|
-
debug(`finished importing ${isoDate(date)}, offset ${offset}`);
|
|
82
|
-
return allEvents;
|
|
83
|
-
});
|
package/dist/importEvent.js
DELETED
|
@@ -1,214 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { sql } from 'kysely';
|
|
11
|
-
import { db, pool } from './db.js';
|
|
12
|
-
/**
|
|
13
|
-
*
|
|
14
|
-
* @param {Client} client
|
|
15
|
-
* @param {import("types").Event} event
|
|
16
|
-
*
|
|
17
|
-
* @return {Promise<void>}
|
|
18
|
-
*/
|
|
19
|
-
export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
|
|
20
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
|
|
21
|
-
// Build a sanitized, typed data object to reduce drift and ensure defaults in one place
|
|
22
|
-
const eventData = {
|
|
23
|
-
action_id: (_a = event.action_id) !== null && _a !== void 0 ? _a : '',
|
|
24
|
-
action_timestamp: event.action_timestamp
|
|
25
|
-
? new Date(event.action_timestamp)
|
|
26
|
-
: new Date(),
|
|
27
|
-
idsite: (_b = event.idsite) !== null && _b !== void 0 ? _b : '',
|
|
28
|
-
idvisit: (_c = event.idvisit) !== null && _c !== void 0 ? _c : '',
|
|
29
|
-
actions: (_d = event.actions) !== null && _d !== void 0 ? _d : null,
|
|
30
|
-
country: (_e = event.country) !== null && _e !== void 0 ? _e : null,
|
|
31
|
-
region: (_f = event.region) !== null && _f !== void 0 ? _f : null,
|
|
32
|
-
city: (_g = event.city) !== null && _g !== void 0 ? _g : null,
|
|
33
|
-
operatingsystemname: (_h = event.operatingsystemname) !== null && _h !== void 0 ? _h : null,
|
|
34
|
-
devicemodel: (_j = event.devicemodel) !== null && _j !== void 0 ? _j : null,
|
|
35
|
-
devicebrand: (_k = event.devicebrand) !== null && _k !== void 0 ? _k : null,
|
|
36
|
-
visitduration: (_l = event.visitduration) !== null && _l !== void 0 ? _l : null,
|
|
37
|
-
dayssincefirstvisit: (_m = event.dayssincefirstvisit) !== null && _m !== void 0 ? _m : null,
|
|
38
|
-
visitortype: (_o = event.visitortype) !== null && _o !== void 0 ? _o : null,
|
|
39
|
-
sitename: (_p = event.sitename) !== null && _p !== void 0 ? _p : null,
|
|
40
|
-
userid: (_q = event.userid) !== null && _q !== void 0 ? _q : null,
|
|
41
|
-
serverdateprettyfirstaction: event.serverdateprettyfirstaction
|
|
42
|
-
? new Date(event.serverdateprettyfirstaction)
|
|
43
|
-
: null,
|
|
44
|
-
action_type: (_r = event.action_type) !== null && _r !== void 0 ? _r : '',
|
|
45
|
-
action_eventcategory: (_s = event.action_eventcategory) !== null && _s !== void 0 ? _s : '',
|
|
46
|
-
action_eventaction: (_t = event.action_eventaction) !== null && _t !== void 0 ? _t : '',
|
|
47
|
-
action_eventname: (_u = event.action_eventname) !== null && _u !== void 0 ? _u : '',
|
|
48
|
-
action_eventvalue: event.action_eventvalue
|
|
49
|
-
? Number(event.action_eventvalue)
|
|
50
|
-
: 0,
|
|
51
|
-
action_timespent: (_v = event.action_timespent) !== null && _v !== void 0 ? _v : '0',
|
|
52
|
-
usercustomproperties: (_w = event.usercustomproperties) !== null && _w !== void 0 ? _w : null,
|
|
53
|
-
usercustomdimensions: (_x = event.usercustomdimensions) !== null && _x !== void 0 ? _x : null,
|
|
54
|
-
dimension1: (_y = event.dimension1) !== null && _y !== void 0 ? _y : null,
|
|
55
|
-
dimension2: (_z = event.dimension2) !== null && _z !== void 0 ? _z : null,
|
|
56
|
-
dimension3: (_0 = event.dimension3) !== null && _0 !== void 0 ? _0 : null,
|
|
57
|
-
dimension4: (_1 = event.dimension4) !== null && _1 !== void 0 ? _1 : null,
|
|
58
|
-
dimension5: (_2 = event.dimension5) !== null && _2 !== void 0 ? _2 : null,
|
|
59
|
-
dimension6: (_3 = event.dimension6) !== null && _3 !== void 0 ? _3 : null,
|
|
60
|
-
dimension7: (_4 = event.dimension7) !== null && _4 !== void 0 ? _4 : null,
|
|
61
|
-
dimension8: (_5 = event.dimension8) !== null && _5 !== void 0 ? _5 : null,
|
|
62
|
-
dimension9: (_6 = event.dimension9) !== null && _6 !== void 0 ? _6 : null,
|
|
63
|
-
dimension10: (_7 = event.dimension10) !== null && _7 !== void 0 ? _7 : null,
|
|
64
|
-
action_url: (_8 = event.action_url) !== null && _8 !== void 0 ? _8 : null,
|
|
65
|
-
sitesearchkeyword: (_9 = event.sitesearchkeyword) !== null && _9 !== void 0 ? _9 : null,
|
|
66
|
-
action_title: (_10 = event.action_title) !== null && _10 !== void 0 ? _10 : null,
|
|
67
|
-
visitorid: (_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null,
|
|
68
|
-
referrertype: (_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null,
|
|
69
|
-
referrername: (_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null,
|
|
70
|
-
resolution: (_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null
|
|
71
|
-
};
|
|
72
|
-
// Minimal runtime validation for required fields
|
|
73
|
-
if (!eventData.action_id || eventData.action_id.trim().length === 0) {
|
|
74
|
-
throw new Error('importEvent(): action_id is required and cannot be empty');
|
|
75
|
-
}
|
|
76
|
-
if (!(eventData.action_timestamp instanceof Date) ||
|
|
77
|
-
isNaN(eventData.action_timestamp.getTime())) {
|
|
78
|
-
throw new Error('importEvent(): action_timestamp is invalid');
|
|
79
|
-
}
|
|
80
|
-
try {
|
|
81
|
-
if (!pool || typeof pool.connect !== 'function') {
|
|
82
|
-
throw new Error('Database connection pool is invalid or undefined');
|
|
83
|
-
}
|
|
84
|
-
// Keep the stored procedure but centralize mapping to avoid parameter mis-ordering
|
|
85
|
-
yield sql `
|
|
86
|
-
SELECT insert_into_matomo_partitioned(
|
|
87
|
-
${eventData.action_id},
|
|
88
|
-
${eventData.action_timestamp},
|
|
89
|
-
${eventData.idsite},
|
|
90
|
-
${eventData.idvisit},
|
|
91
|
-
${eventData.actions},
|
|
92
|
-
${eventData.country},
|
|
93
|
-
${eventData.region},
|
|
94
|
-
${eventData.city},
|
|
95
|
-
${eventData.operatingsystemname},
|
|
96
|
-
${eventData.devicemodel},
|
|
97
|
-
${eventData.devicebrand},
|
|
98
|
-
${eventData.visitduration},
|
|
99
|
-
${eventData.dayssincefirstvisit},
|
|
100
|
-
${eventData.visitortype},
|
|
101
|
-
${eventData.sitename},
|
|
102
|
-
${eventData.userid},
|
|
103
|
-
${eventData.serverdateprettyfirstaction},
|
|
104
|
-
${eventData.action_type},
|
|
105
|
-
${eventData.action_eventcategory},
|
|
106
|
-
${eventData.action_eventaction},
|
|
107
|
-
${eventData.action_eventname},
|
|
108
|
-
${eventData.action_eventvalue},
|
|
109
|
-
${eventData.action_timespent},
|
|
110
|
-
${eventData.usercustomproperties},
|
|
111
|
-
${eventData.usercustomdimensions},
|
|
112
|
-
${eventData.dimension1},
|
|
113
|
-
${eventData.dimension2},
|
|
114
|
-
${eventData.dimension3},
|
|
115
|
-
${eventData.dimension4},
|
|
116
|
-
${eventData.dimension5},
|
|
117
|
-
${eventData.dimension6},
|
|
118
|
-
${eventData.dimension7},
|
|
119
|
-
${eventData.dimension8},
|
|
120
|
-
${eventData.dimension9},
|
|
121
|
-
${eventData.dimension10},
|
|
122
|
-
${eventData.action_url},
|
|
123
|
-
${eventData.sitesearchkeyword},
|
|
124
|
-
${eventData.action_title},
|
|
125
|
-
${eventData.visitorid},
|
|
126
|
-
${eventData.referrertype},
|
|
127
|
-
${eventData.referrername},
|
|
128
|
-
${eventData.resolution}
|
|
129
|
-
)
|
|
130
|
-
`.execute(db);
|
|
131
|
-
}
|
|
132
|
-
catch (err) {
|
|
133
|
-
// Add context for troubleshooting
|
|
134
|
-
const minimalContext = {
|
|
135
|
-
action_id: eventData.action_id,
|
|
136
|
-
action_timestamp: eventData.action_timestamp,
|
|
137
|
-
idsite: eventData.idsite,
|
|
138
|
-
idvisit: eventData.idvisit
|
|
139
|
-
};
|
|
140
|
-
console.error('importEvent(): failed to insert event', minimalContext);
|
|
141
|
-
// Log error details but avoid exposing sensitive information
|
|
142
|
-
console.error('importEvent(): error', err instanceof Error ? err.message : 'Unknown error');
|
|
143
|
-
throw err;
|
|
144
|
-
}
|
|
145
|
-
});
|
|
146
|
-
const matomoProps = [
|
|
147
|
-
'idSite',
|
|
148
|
-
'idVisit',
|
|
149
|
-
'actions',
|
|
150
|
-
'country',
|
|
151
|
-
'region',
|
|
152
|
-
'city',
|
|
153
|
-
'operatingSystemName',
|
|
154
|
-
'deviceModel',
|
|
155
|
-
'deviceBrand',
|
|
156
|
-
'visitDuration',
|
|
157
|
-
'daysSinceFirstVisit',
|
|
158
|
-
'visitorType',
|
|
159
|
-
'visitorId',
|
|
160
|
-
'referrerType',
|
|
161
|
-
'referrerName',
|
|
162
|
-
'siteName',
|
|
163
|
-
'userId',
|
|
164
|
-
'resolution'
|
|
165
|
-
];
|
|
166
|
-
/** @type Record<string, (a: import("types/matomo-api").ActionDetail) => string | number> */
|
|
167
|
-
const actionProps = {
|
|
168
|
-
action_type: (action) => action.type,
|
|
169
|
-
action_title: (action) => action.title,
|
|
170
|
-
action_eventcategory: (action) => action.eventCategory,
|
|
171
|
-
action_eventaction: (action) => action.eventAction,
|
|
172
|
-
action_eventname: (action) => action.eventName,
|
|
173
|
-
action_eventvalue: (action) => action.eventValue,
|
|
174
|
-
action_timespent: (action) => action.timeSpent,
|
|
175
|
-
action_timestamp: (action) => new Date(action.timestamp * 1000).toISOString(),
|
|
176
|
-
action_url: (action) => action.url,
|
|
177
|
-
sitesearchkeyword: (action) => action.siteSearchKeyword
|
|
178
|
-
};
|
|
179
|
-
export const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
180
|
-
return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
|
|
181
|
-
var _a;
|
|
182
|
-
const usercustomproperties = {};
|
|
183
|
-
for (let k = 1; k < 10; k++) {
|
|
184
|
-
const property = (_a = actionDetail.customVariables) === null || _a === void 0 ? void 0 : _a[k];
|
|
185
|
-
if (!property)
|
|
186
|
-
continue; // max 10 custom variables
|
|
187
|
-
//@ts-expect-error implicit any type
|
|
188
|
-
usercustomproperties[property[`customVariableName${k}`]] =
|
|
189
|
-
//@ts-expect-error implicit any type
|
|
190
|
-
property[`customVariableValue${k}`];
|
|
191
|
-
}
|
|
192
|
-
/** @type {Record<string, string>} */
|
|
193
|
-
const usercustomdimensions = {};
|
|
194
|
-
for (let k = 1; k < 11; k++) {
|
|
195
|
-
const dimension = `dimension${k}`;
|
|
196
|
-
//@ts-expect-error implicit any type
|
|
197
|
-
const value = actionDetail[dimension] || matomoVisit[dimension];
|
|
198
|
-
if (!value)
|
|
199
|
-
continue; // max 10 custom variables
|
|
200
|
-
//@ts-expect-error implicit any type
|
|
201
|
-
usercustomdimensions[dimension] = value;
|
|
202
|
-
}
|
|
203
|
-
const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
|
|
204
|
-
action_id: `${matomoVisit.idVisit}_${actionIndex}`
|
|
205
|
-
})), {
|
|
206
|
-
// custom variables
|
|
207
|
-
usercustomproperties,
|
|
208
|
-
// custom dimensions
|
|
209
|
-
// We keep both for backwards compatibility.
|
|
210
|
-
// Current implementation is flat with one column for each dimension.
|
|
211
|
-
usercustomdimensions }), usercustomdimensions);
|
|
212
|
-
return event;
|
|
213
|
-
});
|
|
214
|
-
};
|
package/dist/index.js
DELETED
|
@@ -1,106 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { eachDayOfInterval } from 'date-fns';
|
|
11
|
-
import startDebug from 'debug';
|
|
12
|
-
import { sql } from 'kysely';
|
|
13
|
-
import pAll from 'p-all';
|
|
14
|
-
import { DESTINATION_TABLE, INITIAL_OFFSET, MATOMO_KEY, MATOMO_SITE, MATOMO_URL } from './config.js';
|
|
15
|
-
import { db } from './db.js';
|
|
16
|
-
import { importDate } from './importDate.js';
|
|
17
|
-
import PiwikClient from './PiwikClient.js';
|
|
18
|
-
const debug = startDebug('index');
|
|
19
|
-
function run(date) {
|
|
20
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
21
|
-
console.log(`🚀 Starting data import process`);
|
|
22
|
-
debug('run, date=' + date);
|
|
23
|
-
console.log(`🔗 Initializing Matomo client`);
|
|
24
|
-
console.log(` - Matomo URL: ${MATOMO_URL}`);
|
|
25
|
-
console.log(` - Matomo Site ID: ${MATOMO_SITE}`);
|
|
26
|
-
const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
|
|
27
|
-
console.log(`📅 Determining reference date for import...`);
|
|
28
|
-
// priority:
|
|
29
|
-
// - optional parameter date
|
|
30
|
-
// - STARTDATE (if FORCE_STARTDATE is true)
|
|
31
|
-
// - last event in the table
|
|
32
|
-
// - STARTDATE (if FORCE_STARTDATE is false)
|
|
33
|
-
// - today
|
|
34
|
-
let referenceDate;
|
|
35
|
-
if (!referenceDate && date) {
|
|
36
|
-
referenceDate = new Date(date);
|
|
37
|
-
console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
|
|
38
|
-
}
|
|
39
|
-
// Check FORCE_STARTDATE mode first
|
|
40
|
-
const forceStartDate = process.env.FORCE_STARTDATE === 'true';
|
|
41
|
-
if (!referenceDate && forceStartDate && process.env.STARTDATE) {
|
|
42
|
-
referenceDate = new Date(process.env.STARTDATE);
|
|
43
|
-
console.log(`✅ FORCE_STARTDATE enabled - Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
|
|
44
|
-
}
|
|
45
|
-
// Only query database if not forcing STARTDATE
|
|
46
|
-
if (!referenceDate && !forceStartDate) {
|
|
47
|
-
console.log(`🔍 Looking for last event in database...`);
|
|
48
|
-
referenceDate = yield findLastEventInMatomo(db);
|
|
49
|
-
if (referenceDate) {
|
|
50
|
-
console.log(`✅ Found last event, starting from: ${referenceDate.toISOString()}`);
|
|
51
|
-
}
|
|
52
|
-
else {
|
|
53
|
-
console.log(`ℹ️ No previous events found in database`);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
// Fallback to STARTDATE if database had no results
|
|
57
|
-
if (!referenceDate && process.env.STARTDATE) {
|
|
58
|
-
referenceDate = new Date(process.env.STARTDATE);
|
|
59
|
-
console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
|
|
60
|
-
}
|
|
61
|
-
if (!referenceDate) {
|
|
62
|
-
referenceDate = new Date(new Date().getTime() - +INITIAL_OFFSET * 24 * 60 * 60 * 1000);
|
|
63
|
-
console.log(`✅ Using default offset (${INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
|
|
64
|
-
}
|
|
65
|
-
const endDate = new Date(new Date().getTime() + 24 * 60 * 60 * 1000);
|
|
66
|
-
const dates = eachDayOfInterval({
|
|
67
|
-
start: referenceDate,
|
|
68
|
-
end: endDate
|
|
69
|
-
});
|
|
70
|
-
console.log(`📊 Import date range determined:`);
|
|
71
|
-
console.log(` - Start date: ${dates[0].toISOString()}`);
|
|
72
|
-
console.log(` - End date: ${endDate.toISOString()}`);
|
|
73
|
-
console.log(` - Total days to process: ${dates.length}`);
|
|
74
|
-
debug(`import starting at : ${dates[0].toISOString()}`);
|
|
75
|
-
console.log(`🔄 Starting sequential import for each date...`);
|
|
76
|
-
// for each date, serial-import data
|
|
77
|
-
const res = yield pAll(dates.map((date, index) => () => {
|
|
78
|
-
console.log(`📅 Processing date ${index + 1}/${dates.length}: ${date.toISOString().split('T')[0]}`);
|
|
79
|
-
return importDate(piwik.api.bind(piwik), date);
|
|
80
|
-
}), { concurrency: 1, stopOnError: true });
|
|
81
|
-
const totalEvents = res.flat().length;
|
|
82
|
-
console.log(`✅ Import process completed`);
|
|
83
|
-
console.log(`📈 Summary:`);
|
|
84
|
-
console.log(` - Days processed: ${dates.length}`);
|
|
85
|
-
console.log(` - Total events imported: ${totalEvents}`);
|
|
86
|
-
debug('close');
|
|
87
|
-
return res;
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
function findLastEventInMatomo(db) {
|
|
91
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
92
|
-
const latest = yield db
|
|
93
|
-
.selectFrom(DESTINATION_TABLE)
|
|
94
|
-
.select(sql `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
|
|
95
|
-
.orderBy('action_timestamp', 'desc')
|
|
96
|
-
.limit(1)
|
|
97
|
-
.executeTakeFirst();
|
|
98
|
-
if (latest) {
|
|
99
|
-
// check from the day before just to be sure we have all events
|
|
100
|
-
const date = new Date(new Date(latest.action_timestamp).getTime() - 2 * 24 * 60 * 60 * 1000);
|
|
101
|
-
return date;
|
|
102
|
-
}
|
|
103
|
-
return null;
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
export default run;
|
package/dist/migrate-down.js
DELETED
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { promises as fs } from 'fs';
|
|
11
|
-
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
-
import * as path from 'path';
|
|
13
|
-
import { db } from './db.js';
|
|
14
|
-
function migrateDown() {
|
|
15
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
16
|
-
const migrator = new Migrator({
|
|
17
|
-
db,
|
|
18
|
-
provider: new FileMigrationProvider({
|
|
19
|
-
fs,
|
|
20
|
-
path,
|
|
21
|
-
migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
|
|
22
|
-
})
|
|
23
|
-
});
|
|
24
|
-
const { error, results } = yield migrator.migrateDown();
|
|
25
|
-
results === null || results === void 0 ? void 0 : results.forEach((it) => {
|
|
26
|
-
if (it.status === 'Success') {
|
|
27
|
-
console.log(`down migration "${it.migrationName}" was executed successfully`);
|
|
28
|
-
}
|
|
29
|
-
else if (it.status === 'Error') {
|
|
30
|
-
console.error(`failed to execute down migration "${it.migrationName}"`);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
if (error) {
|
|
34
|
-
console.error('failed to down migrate');
|
|
35
|
-
console.error(error);
|
|
36
|
-
process.exit(1);
|
|
37
|
-
}
|
|
38
|
-
yield db.destroy();
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
migrateDown();
|
package/dist/migrate-latest.js
DELETED
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { promises as fs } from 'fs';
|
|
11
|
-
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
-
import * as path from 'path';
|
|
13
|
-
import { MATOMO_TABLE_NAME } from './config.js';
|
|
14
|
-
import { db } from './db.js';
|
|
15
|
-
function migrateToLatest() {
|
|
16
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
17
|
-
console.log(`Starting migrate to latest`);
|
|
18
|
-
try {
|
|
19
|
-
const migrator = new Migrator({
|
|
20
|
-
db,
|
|
21
|
-
provider: new FileMigrationProvider({
|
|
22
|
-
fs,
|
|
23
|
-
path,
|
|
24
|
-
migrationFolder: path.join(path.dirname(new URL(import.meta.url).pathname), 'migrations')
|
|
25
|
-
}),
|
|
26
|
-
// allow to have mutliple migratable instances in a single schema
|
|
27
|
-
migrationTableName: `${MATOMO_TABLE_NAME}_migration`,
|
|
28
|
-
migrationLockTableName: `${MATOMO_TABLE_NAME}_migration_lock`
|
|
29
|
-
});
|
|
30
|
-
const { error, results } = yield migrator.migrateToLatest();
|
|
31
|
-
results === null || results === void 0 ? void 0 : results.forEach((it) => {
|
|
32
|
-
if (it.status === 'Success') {
|
|
33
|
-
console.log(`migration "${it.migrationName}" was executed successfully`);
|
|
34
|
-
}
|
|
35
|
-
else if (it.status === 'Error') {
|
|
36
|
-
console.error(`failed to execute migration "${it.migrationName}"`);
|
|
37
|
-
}
|
|
38
|
-
});
|
|
39
|
-
if (error) {
|
|
40
|
-
console.error('failed to migrate');
|
|
41
|
-
console.error(error);
|
|
42
|
-
process.exit(1);
|
|
43
|
-
}
|
|
44
|
-
else if (!(results === null || results === void 0 ? void 0 : results.length)) {
|
|
45
|
-
console.log('No migration to run');
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
catch (uncaughtError) {
|
|
49
|
-
console.error('UNCAUGHT ERROR during migration:');
|
|
50
|
-
console.error('Error message:', uncaughtError instanceof Error
|
|
51
|
-
? uncaughtError.message
|
|
52
|
-
: String(uncaughtError));
|
|
53
|
-
console.error('Error stack:', uncaughtError instanceof Error
|
|
54
|
-
? uncaughtError.stack
|
|
55
|
-
: 'No stack trace available');
|
|
56
|
-
console.error('Full error object:', uncaughtError);
|
|
57
|
-
process.exit(1);
|
|
58
|
-
}
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
export default migrateToLatest;
|
|
62
|
-
export function startMigration() {
|
|
63
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
64
|
-
yield migrateToLatest();
|
|
65
|
-
// Don't destroy the db connection here since the main application will need it
|
|
66
|
-
});
|
|
67
|
-
}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { sql } from 'kysely';
|
|
11
|
-
const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
|
|
12
|
-
export function up(db) {
|
|
13
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
14
|
-
yield db.schema
|
|
15
|
-
.createTable(MATOMO_TABLE_NAME)
|
|
16
|
-
.ifNotExists()
|
|
17
|
-
.addColumn('action_id', 'text', (col) => col.unique().notNull())
|
|
18
|
-
.addColumn('idsite', 'text')
|
|
19
|
-
.addColumn('idvisit', 'text')
|
|
20
|
-
.addColumn('actions', 'text')
|
|
21
|
-
.addColumn('country', 'text')
|
|
22
|
-
.addColumn('region', 'text')
|
|
23
|
-
.addColumn('city', 'text')
|
|
24
|
-
.addColumn('operatingsystemname', 'text')
|
|
25
|
-
.addColumn('devicemodel', 'text')
|
|
26
|
-
.addColumn('devicebrand', 'text')
|
|
27
|
-
.addColumn('visitduration', 'text')
|
|
28
|
-
.addColumn('dayssincefirstvisit', 'text')
|
|
29
|
-
.addColumn('visitortype', 'text')
|
|
30
|
-
.addColumn('sitename', 'text')
|
|
31
|
-
.addColumn('userid', 'text')
|
|
32
|
-
.addColumn('serverdateprettyfirstaction', 'date')
|
|
33
|
-
.addColumn('action_type', 'text')
|
|
34
|
-
.addColumn('action_eventcategory', 'text')
|
|
35
|
-
.addColumn('action_eventaction', 'text')
|
|
36
|
-
.addColumn('action_eventname', 'text')
|
|
37
|
-
.addColumn('action_eventvalue', 'numeric')
|
|
38
|
-
.addColumn('action_timespent', 'text')
|
|
39
|
-
.addColumn('action_timestamp', 'timestamptz', (col) => col.defaultTo(sql `now()`))
|
|
40
|
-
.addColumn('usercustomproperties', 'json')
|
|
41
|
-
.addColumn('usercustomdimensions', 'json')
|
|
42
|
-
.addColumn('dimension1', 'text')
|
|
43
|
-
.addColumn('dimension2', 'text')
|
|
44
|
-
.addColumn('dimension3', 'text')
|
|
45
|
-
.addColumn('dimension4', 'text')
|
|
46
|
-
.addColumn('dimension5', 'text')
|
|
47
|
-
.addColumn('dimension6', 'text')
|
|
48
|
-
.addColumn('dimension7', 'text')
|
|
49
|
-
.addColumn('dimension8', 'text')
|
|
50
|
-
.addColumn('dimension9', 'text')
|
|
51
|
-
.addColumn('dimension10', 'text')
|
|
52
|
-
.addColumn('action_url', 'text')
|
|
53
|
-
.addColumn('sitesearchkeyword', 'text')
|
|
54
|
-
.addColumn('action_title', 'text')
|
|
55
|
-
.addColumn('visitorid', 'text')
|
|
56
|
-
.addColumn('referrertype', 'text')
|
|
57
|
-
.addColumn('referrername', 'text')
|
|
58
|
-
.execute();
|
|
59
|
-
});
|
|
60
|
-
}
|
|
61
|
-
export function down(db) {
|
|
62
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
63
|
-
yield db.schema.dropTable(MATOMO_TABLE_NAME).execute();
|
|
64
|
-
});
|
|
65
|
-
}
|