@socialgouv/matomo-postgres 2.1.0 → 2.2.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/index.js +14 -16
- package/dist/PiwikClient.js +12 -34
- package/dist/__tests__/importDate.test.js +24 -27
- package/dist/__tests__/importEvent.test.js +7 -13
- package/dist/__tests__/run.test.js +35 -40
- package/dist/config.js +8 -10
- package/dist/db.js +14 -19
- package/dist/importDate.js +26 -34
- package/dist/importEvent.js +91 -40
- package/dist/index.js +70 -44
- package/dist/migrate-down.js +17 -39
- package/dist/migrate-latest.js +22 -46
- package/dist/migrations/20230301-01-initial.js +47 -52
- package/dist/migrations/20230301-02-indexes.js +41 -46
- package/dist/migrations/20250425-01-add-resolution.js +11 -10
- package/dist/migrations/20250715-01-weekly-partitioning.js +357 -0
- package/package.json +16 -2
package/dist/importEvent.js
CHANGED
|
@@ -1,42 +1,92 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { sql } from 'kysely';
|
|
11
|
+
import { db } from './db.js';
|
|
6
12
|
/**
|
|
7
13
|
*
|
|
8
14
|
* @param {Client} client
|
|
9
15
|
* @param {import("types").Event} event
|
|
10
16
|
*
|
|
11
|
-
* @return {Promise<
|
|
17
|
+
* @return {Promise<void>}
|
|
12
18
|
*/
|
|
13
|
-
const importEvent = (event) =>
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
export const importEvent = (event) => __awaiter(void 0, void 0, void 0, function* () {
|
|
20
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14;
|
|
21
|
+
// Use the stored procedure for safe insertion with automatic partition creation
|
|
22
|
+
yield sql `
|
|
23
|
+
SELECT insert_into_matomo_partitioned(
|
|
24
|
+
${(_a = event.action_id) !== null && _a !== void 0 ? _a : ''},
|
|
25
|
+
${event.action_timestamp ? new Date(event.action_timestamp) : new Date()},
|
|
26
|
+
${(_b = event.idsite) !== null && _b !== void 0 ? _b : ''},
|
|
27
|
+
${(_c = event.idvisit) !== null && _c !== void 0 ? _c : ''},
|
|
28
|
+
${(_d = event.actions) !== null && _d !== void 0 ? _d : null},
|
|
29
|
+
${(_e = event.country) !== null && _e !== void 0 ? _e : null},
|
|
30
|
+
${(_f = event.region) !== null && _f !== void 0 ? _f : null},
|
|
31
|
+
${(_g = event.city) !== null && _g !== void 0 ? _g : null},
|
|
32
|
+
${(_h = event.operatingsystemname) !== null && _h !== void 0 ? _h : null},
|
|
33
|
+
${(_j = event.devicemodel) !== null && _j !== void 0 ? _j : null},
|
|
34
|
+
${(_k = event.devicebrand) !== null && _k !== void 0 ? _k : null},
|
|
35
|
+
${(_l = event.visitduration) !== null && _l !== void 0 ? _l : null},
|
|
36
|
+
${(_m = event.dayssincefirstvisit) !== null && _m !== void 0 ? _m : null},
|
|
37
|
+
${(_o = event.visitortype) !== null && _o !== void 0 ? _o : null},
|
|
38
|
+
${(_p = event.sitename) !== null && _p !== void 0 ? _p : null},
|
|
39
|
+
${(_q = event.userid) !== null && _q !== void 0 ? _q : null},
|
|
40
|
+
${event.serverdateprettyfirstaction
|
|
41
|
+
? new Date(event.serverdateprettyfirstaction)
|
|
42
|
+
: null},
|
|
43
|
+
${(_r = event.action_type) !== null && _r !== void 0 ? _r : ''},
|
|
44
|
+
${(_s = event.action_eventcategory) !== null && _s !== void 0 ? _s : ''},
|
|
45
|
+
${(_t = event.action_eventaction) !== null && _t !== void 0 ? _t : ''},
|
|
46
|
+
${(_u = event.action_eventname) !== null && _u !== void 0 ? _u : ''},
|
|
47
|
+
${event.action_eventvalue ? Number(event.action_eventvalue) : 0},
|
|
48
|
+
${(_v = event.action_timespent) !== null && _v !== void 0 ? _v : '0'},
|
|
49
|
+
${(_w = event.usercustomproperties) !== null && _w !== void 0 ? _w : null},
|
|
50
|
+
${(_x = event.usercustomdimensions) !== null && _x !== void 0 ? _x : null},
|
|
51
|
+
${(_y = event.dimension1) !== null && _y !== void 0 ? _y : null},
|
|
52
|
+
${(_z = event.dimension2) !== null && _z !== void 0 ? _z : null},
|
|
53
|
+
${(_0 = event.dimension3) !== null && _0 !== void 0 ? _0 : null},
|
|
54
|
+
${(_1 = event.dimension4) !== null && _1 !== void 0 ? _1 : null},
|
|
55
|
+
${(_2 = event.dimension5) !== null && _2 !== void 0 ? _2 : null},
|
|
56
|
+
${(_3 = event.dimension6) !== null && _3 !== void 0 ? _3 : null},
|
|
57
|
+
${(_4 = event.dimension7) !== null && _4 !== void 0 ? _4 : null},
|
|
58
|
+
${(_5 = event.dimension8) !== null && _5 !== void 0 ? _5 : null},
|
|
59
|
+
${(_6 = event.dimension9) !== null && _6 !== void 0 ? _6 : null},
|
|
60
|
+
${(_7 = event.dimension10) !== null && _7 !== void 0 ? _7 : null},
|
|
61
|
+
${(_8 = event.action_url) !== null && _8 !== void 0 ? _8 : null},
|
|
62
|
+
${(_9 = event.sitesearchkeyword) !== null && _9 !== void 0 ? _9 : null},
|
|
63
|
+
${(_10 = event.action_title) !== null && _10 !== void 0 ? _10 : null},
|
|
64
|
+
${(_11 = event.visitorid) !== null && _11 !== void 0 ? _11 : null},
|
|
65
|
+
${(_12 = event.referrertype) !== null && _12 !== void 0 ? _12 : null},
|
|
66
|
+
${(_13 = event.referrername) !== null && _13 !== void 0 ? _13 : null},
|
|
67
|
+
${(_14 = event.resolution) !== null && _14 !== void 0 ? _14 : null}
|
|
68
|
+
)
|
|
69
|
+
`.execute(db);
|
|
70
|
+
});
|
|
21
71
|
const matomoProps = [
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
72
|
+
'idSite',
|
|
73
|
+
'idVisit',
|
|
74
|
+
'actions',
|
|
75
|
+
'country',
|
|
76
|
+
'region',
|
|
77
|
+
'city',
|
|
78
|
+
'operatingSystemName',
|
|
79
|
+
'deviceModel',
|
|
80
|
+
'deviceBrand',
|
|
81
|
+
'visitDuration',
|
|
82
|
+
'daysSinceFirstVisit',
|
|
83
|
+
'visitorType',
|
|
84
|
+
'visitorId',
|
|
85
|
+
'referrerType',
|
|
86
|
+
'referrerName',
|
|
87
|
+
'siteName',
|
|
88
|
+
'userId',
|
|
89
|
+
'resolution'
|
|
40
90
|
];
|
|
41
91
|
/** @type Record<string, (a: import("types/matomo-api").ActionDetail) => string | number> */
|
|
42
92
|
const actionProps = {
|
|
@@ -49,31 +99,33 @@ const actionProps = {
|
|
|
49
99
|
action_timespent: (action) => action.timeSpent,
|
|
50
100
|
action_timestamp: (action) => new Date(action.timestamp * 1000).toISOString(),
|
|
51
101
|
action_url: (action) => action.url,
|
|
52
|
-
sitesearchkeyword: (action) => action.siteSearchKeyword
|
|
102
|
+
sitesearchkeyword: (action) => action.siteSearchKeyword
|
|
53
103
|
};
|
|
54
|
-
const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
104
|
+
export const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
55
105
|
return matomoVisit.actionDetails.map((actionDetail, actionIndex) => {
|
|
56
106
|
const usercustomproperties = {};
|
|
57
107
|
for (let k = 1; k < 10; k++) {
|
|
58
108
|
const property = actionDetail.customVariables && actionDetail.customVariables[k];
|
|
59
109
|
if (!property)
|
|
60
110
|
continue; // max 10 custom variables
|
|
61
|
-
//@ts-
|
|
62
|
-
usercustomproperties[property[`customVariableName${k}`]] =
|
|
111
|
+
//@ts-expect-error implicit any type
|
|
112
|
+
usercustomproperties[property[`customVariableName${k}`]] =
|
|
113
|
+
//@ts-expect-error implicit any type
|
|
114
|
+
property[`customVariableValue${k}`];
|
|
63
115
|
}
|
|
64
116
|
/** @type {Record<string, string>} */
|
|
65
117
|
const usercustomdimensions = {};
|
|
66
118
|
for (let k = 1; k < 11; k++) {
|
|
67
119
|
const dimension = `dimension${k}`;
|
|
68
|
-
//@ts-
|
|
120
|
+
//@ts-expect-error implicit any type
|
|
69
121
|
const value = actionDetail[dimension] || matomoVisit[dimension];
|
|
70
122
|
if (!value)
|
|
71
123
|
continue; // max 10 custom variables
|
|
72
|
-
//@ts-
|
|
124
|
+
//@ts-expect-error implicit any type
|
|
73
125
|
usercustomdimensions[dimension] = value;
|
|
74
126
|
}
|
|
75
127
|
const event = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, matomoProps.reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: matomoVisit[prop] })), {})), { serverdateprettyfirstaction: new Date((matomoVisit.firstActionTimestamp || 0) * 1000).toISOString() }), Object.keys(actionProps).reduce((a, prop) => (Object.assign(Object.assign({}, a), { [prop.toLowerCase()]: actionProps[prop](actionDetail) })), {
|
|
76
|
-
action_id: `${matomoVisit.idVisit}_${actionIndex}
|
|
128
|
+
action_id: `${matomoVisit.idVisit}_${actionIndex}`
|
|
77
129
|
})), {
|
|
78
130
|
// custom variables
|
|
79
131
|
usercustomproperties,
|
|
@@ -84,4 +136,3 @@ const getEventsFromMatomoVisit = (matomoVisit) => {
|
|
|
84
136
|
return event;
|
|
85
137
|
});
|
|
86
138
|
};
|
|
87
|
-
exports.getEventsFromMatomoVisit = getEventsFromMatomoVisit;
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,75 +7,102 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
const
|
|
20
|
-
const db_1 = require("./db");
|
|
21
|
-
const config_1 = require("./config");
|
|
22
|
-
const importDate_1 = require("./importDate");
|
|
23
|
-
const debug = (0, debug_1.default)("index");
|
|
10
|
+
import { eachDayOfInterval } from 'date-fns';
|
|
11
|
+
import startDebug from 'debug';
|
|
12
|
+
import { sql } from 'kysely';
|
|
13
|
+
import pAll from 'p-all';
|
|
14
|
+
import { DESTINATION_TABLE, INITIAL_OFFSET, MATOMO_KEY, MATOMO_SITE, MATOMO_URL, PGDATABASE } from './config.js';
|
|
15
|
+
import { db } from './db.js';
|
|
16
|
+
import { importDate } from './importDate.js';
|
|
17
|
+
import PiwikClient from './PiwikClient.js';
|
|
18
|
+
const debug = startDebug('index');
|
|
24
19
|
function run(date) {
|
|
25
20
|
return __awaiter(this, void 0, void 0, function* () {
|
|
26
|
-
|
|
27
|
-
|
|
21
|
+
console.log(`🚀 Starting data import process`);
|
|
22
|
+
debug('run, date=' + date);
|
|
23
|
+
console.log(`🔗 Initializing Matomo client`);
|
|
24
|
+
console.log(` - Matomo URL: ${MATOMO_URL}`);
|
|
25
|
+
console.log(` - Matomo Site ID: ${MATOMO_SITE}`);
|
|
26
|
+
const piwik = new PiwikClient(MATOMO_URL, MATOMO_KEY);
|
|
27
|
+
console.log(`📅 Determining reference date for import...`);
|
|
28
28
|
// priority:
|
|
29
29
|
// - optional parameter date
|
|
30
30
|
// - last event in the table
|
|
31
31
|
// - optional env.STARTDATE
|
|
32
32
|
// - today
|
|
33
33
|
let referenceDate;
|
|
34
|
-
if (!referenceDate && date)
|
|
34
|
+
if (!referenceDate && date) {
|
|
35
35
|
referenceDate = new Date(date);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (!referenceDate && process.env.STARTDATE)
|
|
36
|
+
console.log(`✅ Using provided date parameter: ${referenceDate.toISOString()}`);
|
|
37
|
+
}
|
|
38
|
+
if (!referenceDate && process.env.STARTDATE) {
|
|
39
39
|
referenceDate = new Date(process.env.STARTDATE);
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
40
|
+
console.log(`✅ Using STARTDATE environment variable: ${referenceDate.toISOString()}`);
|
|
41
|
+
}
|
|
42
|
+
if (!referenceDate) {
|
|
43
|
+
console.log(`🔍 Looking for last event in database...`);
|
|
44
|
+
referenceDate = yield findLastEventInMatomo(db);
|
|
45
|
+
if (referenceDate) {
|
|
46
|
+
console.log(`✅ Found last event, starting from: ${referenceDate.toISOString()}`);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
console.log(`ℹ️ No previous events found in database`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (!referenceDate) {
|
|
53
|
+
referenceDate = new Date(new Date().getTime() - +INITIAL_OFFSET * 24 * 60 * 60 * 1000);
|
|
54
|
+
console.log(`✅ Using default offset (${INITIAL_OFFSET} days ago): ${referenceDate.toISOString()}`);
|
|
55
|
+
}
|
|
56
|
+
const endDate = new Date(new Date().getTime() + 24 * 60 * 60 * 1000);
|
|
57
|
+
const dates = eachDayOfInterval({
|
|
43
58
|
start: referenceDate,
|
|
44
|
-
end:
|
|
59
|
+
end: endDate
|
|
45
60
|
});
|
|
61
|
+
console.log(`📊 Import date range determined:`);
|
|
62
|
+
console.log(` - Start date: ${dates[0].toISOString()}`);
|
|
63
|
+
console.log(` - End date: ${endDate.toISOString()}`);
|
|
64
|
+
console.log(` - Total days to process: ${dates.length}`);
|
|
46
65
|
debug(`import starting at : ${dates[0].toISOString()}`);
|
|
66
|
+
console.log(`🔄 Starting sequential import for each date...`);
|
|
47
67
|
// for each date, serial-import data
|
|
48
|
-
const res = yield (
|
|
49
|
-
|
|
68
|
+
const res = yield pAll(dates.map((date, index) => () => {
|
|
69
|
+
console.log(`📅 Processing date ${index + 1}/${dates.length}: ${date.toISOString().split('T')[0]}`);
|
|
70
|
+
return importDate(piwik.api.bind(piwik), date);
|
|
71
|
+
}), { concurrency: 1, stopOnError: true });
|
|
72
|
+
const totalEvents = res.flat().length;
|
|
73
|
+
console.log(`✅ Import process completed`);
|
|
74
|
+
console.log(`📈 Summary:`);
|
|
75
|
+
console.log(` - Days processed: ${dates.length}`);
|
|
76
|
+
console.log(` - Total events imported: ${totalEvents}`);
|
|
77
|
+
debug('close');
|
|
50
78
|
return res;
|
|
51
79
|
});
|
|
52
80
|
}
|
|
53
|
-
exports.default = run;
|
|
54
|
-
if (require.main === module) {
|
|
55
|
-
(() => __awaiter(void 0, void 0, void 0, function* () {
|
|
56
|
-
if (!config_1.MATOMO_SITE)
|
|
57
|
-
return console.error("Missing env MATOMO_SITE");
|
|
58
|
-
if (!config_1.MATOMO_KEY)
|
|
59
|
-
return console.error("Missing env MATOMO_KEY");
|
|
60
|
-
if (!config_1.PGDATABASE)
|
|
61
|
-
return console.error("Missing env PGDATABASE");
|
|
62
|
-
yield run();
|
|
63
|
-
debug("run finished");
|
|
64
|
-
db_1.db.destroy();
|
|
65
|
-
}))();
|
|
66
|
-
}
|
|
67
81
|
function findLastEventInMatomo(db) {
|
|
68
82
|
return __awaiter(this, void 0, void 0, function* () {
|
|
69
83
|
const latest = yield db
|
|
70
|
-
.selectFrom(
|
|
71
|
-
.select(
|
|
72
|
-
.orderBy(
|
|
84
|
+
.selectFrom(DESTINATION_TABLE)
|
|
85
|
+
.select(sql `action_timestamp at time zone 'UTC'`.as('action_timestamp'))
|
|
86
|
+
.orderBy('action_timestamp', 'desc')
|
|
73
87
|
.limit(1)
|
|
74
88
|
.executeTakeFirst();
|
|
75
89
|
if (latest) {
|
|
76
|
-
// check from the day before just to be sure we
|
|
90
|
+
// check from the day before just to be sure we have all events
|
|
77
91
|
const date = new Date(new Date(latest.action_timestamp).getTime() - 2 * 24 * 60 * 60 * 1000);
|
|
78
92
|
return date;
|
|
79
93
|
}
|
|
80
94
|
return null;
|
|
81
95
|
});
|
|
82
96
|
}
|
|
97
|
+
export default run;
|
|
98
|
+
(() => __awaiter(void 0, void 0, void 0, function* () {
|
|
99
|
+
if (!MATOMO_SITE)
|
|
100
|
+
return console.error('Missing env MATOMO_SITE');
|
|
101
|
+
if (!MATOMO_KEY)
|
|
102
|
+
return console.error('Missing env MATOMO_KEY');
|
|
103
|
+
if (!PGDATABASE)
|
|
104
|
+
return console.error('Missing env PGDATABASE');
|
|
105
|
+
yield run();
|
|
106
|
+
debug('run finished');
|
|
107
|
+
db.destroy();
|
|
108
|
+
}))();
|
package/dist/migrate-down.js
CHANGED
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -31,36 +7,38 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
31
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
8
|
});
|
|
33
9
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const
|
|
10
|
+
import { promises as fs } from 'fs';
|
|
11
|
+
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
import { fileURLToPath } from 'url';
|
|
14
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
15
|
+
const __dirname = path.dirname(__filename);
|
|
16
|
+
import { db } from './db.js';
|
|
39
17
|
function migrateDown() {
|
|
40
18
|
return __awaiter(this, void 0, void 0, function* () {
|
|
41
|
-
const migrator = new
|
|
42
|
-
db
|
|
43
|
-
provider: new
|
|
44
|
-
fs
|
|
19
|
+
const migrator = new Migrator({
|
|
20
|
+
db,
|
|
21
|
+
provider: new FileMigrationProvider({
|
|
22
|
+
fs,
|
|
45
23
|
path,
|
|
46
|
-
migrationFolder: __dirname +
|
|
47
|
-
})
|
|
24
|
+
migrationFolder: __dirname + '/migrations'
|
|
25
|
+
})
|
|
48
26
|
});
|
|
49
27
|
const { error, results } = yield migrator.migrateDown();
|
|
50
28
|
results === null || results === void 0 ? void 0 : results.forEach((it) => {
|
|
51
|
-
if (it.status ===
|
|
29
|
+
if (it.status === 'Success') {
|
|
52
30
|
console.log(`down migration "${it.migrationName}" was executed successfully`);
|
|
53
31
|
}
|
|
54
|
-
else if (it.status ===
|
|
32
|
+
else if (it.status === 'Error') {
|
|
55
33
|
console.error(`failed to execute down migration "${it.migrationName}"`);
|
|
56
34
|
}
|
|
57
35
|
});
|
|
58
36
|
if (error) {
|
|
59
|
-
console.error(
|
|
37
|
+
console.error('failed to down migrate');
|
|
60
38
|
console.error(error);
|
|
61
39
|
process.exit(1);
|
|
62
40
|
}
|
|
63
|
-
yield
|
|
41
|
+
yield db.destroy();
|
|
64
42
|
});
|
|
65
43
|
}
|
|
66
44
|
migrateDown();
|
package/dist/migrate-latest.js
CHANGED
|
@@ -1,27 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -31,53 +7,53 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
31
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
8
|
});
|
|
33
9
|
};
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const
|
|
39
|
-
const
|
|
10
|
+
import { promises as fs } from 'fs';
|
|
11
|
+
import { FileMigrationProvider, Migrator } from 'kysely';
|
|
12
|
+
import * as path from 'path';
|
|
13
|
+
import { fileURLToPath } from 'url';
|
|
14
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
15
|
+
const __dirname = path.dirname(__filename);
|
|
16
|
+
import { MATOMO_TABLE_NAME } from './config.js';
|
|
17
|
+
import { db } from './db.js';
|
|
40
18
|
function migrateToLatest() {
|
|
41
19
|
return __awaiter(this, void 0, void 0, function* () {
|
|
42
|
-
const migrator = new
|
|
43
|
-
db
|
|
44
|
-
provider: new
|
|
45
|
-
fs
|
|
20
|
+
const migrator = new Migrator({
|
|
21
|
+
db,
|
|
22
|
+
provider: new FileMigrationProvider({
|
|
23
|
+
fs,
|
|
46
24
|
path,
|
|
47
|
-
migrationFolder: __dirname +
|
|
25
|
+
migrationFolder: __dirname + '/migrations'
|
|
48
26
|
}),
|
|
49
27
|
// allow to have mutliple migratable instances in a single schema
|
|
50
|
-
migrationTableName: `${
|
|
51
|
-
migrationLockTableName: `${
|
|
28
|
+
migrationTableName: `${MATOMO_TABLE_NAME}_migration`,
|
|
29
|
+
migrationLockTableName: `${MATOMO_TABLE_NAME}_migration_lock`
|
|
52
30
|
});
|
|
53
31
|
const { error, results } = yield migrator.migrateToLatest();
|
|
54
32
|
results === null || results === void 0 ? void 0 : results.forEach((it) => {
|
|
55
|
-
if (it.status ===
|
|
33
|
+
if (it.status === 'Success') {
|
|
56
34
|
console.log(`migration "${it.migrationName}" was executed successfully`);
|
|
57
35
|
}
|
|
58
|
-
else if (it.status ===
|
|
36
|
+
else if (it.status === 'Error') {
|
|
59
37
|
console.error(`failed to execute migration "${it.migrationName}"`);
|
|
60
38
|
}
|
|
61
39
|
});
|
|
62
40
|
if (error) {
|
|
63
|
-
console.error(
|
|
41
|
+
console.error('failed to migrate');
|
|
64
42
|
console.error(error);
|
|
65
43
|
process.exit(1);
|
|
66
44
|
}
|
|
67
45
|
else {
|
|
68
46
|
if (!(results === null || results === void 0 ? void 0 : results.length)) {
|
|
69
|
-
console.log(
|
|
47
|
+
console.log('No migration to run');
|
|
70
48
|
}
|
|
71
49
|
}
|
|
72
50
|
});
|
|
73
51
|
}
|
|
74
|
-
|
|
52
|
+
export default migrateToLatest;
|
|
75
53
|
function start() {
|
|
76
54
|
return __awaiter(this, void 0, void 0, function* () {
|
|
77
55
|
yield migrateToLatest();
|
|
78
|
-
yield
|
|
56
|
+
yield db.destroy();
|
|
79
57
|
});
|
|
80
58
|
}
|
|
81
|
-
|
|
82
|
-
start();
|
|
83
|
-
}
|
|
59
|
+
start();
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
"use strict";
|
|
2
1
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
2
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
3
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -8,63 +7,59 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
8
7
|
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
8
|
});
|
|
10
9
|
};
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
const DESTINATION_TABLE = process.env.DESTINATION_TABLE || "matomo";
|
|
15
|
-
function up(db) {
|
|
10
|
+
import { sql } from 'kysely';
|
|
11
|
+
const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
|
|
12
|
+
export function up(db) {
|
|
16
13
|
return __awaiter(this, void 0, void 0, function* () {
|
|
17
14
|
yield db.schema
|
|
18
|
-
.createTable(
|
|
15
|
+
.createTable(MATOMO_TABLE_NAME)
|
|
19
16
|
.ifNotExists()
|
|
20
|
-
.addColumn(
|
|
21
|
-
.addColumn(
|
|
22
|
-
.addColumn(
|
|
23
|
-
.addColumn(
|
|
24
|
-
.addColumn(
|
|
25
|
-
.addColumn(
|
|
26
|
-
.addColumn(
|
|
27
|
-
.addColumn(
|
|
28
|
-
.addColumn(
|
|
29
|
-
.addColumn(
|
|
30
|
-
.addColumn(
|
|
31
|
-
.addColumn(
|
|
32
|
-
.addColumn(
|
|
33
|
-
.addColumn(
|
|
34
|
-
.addColumn(
|
|
35
|
-
.addColumn(
|
|
36
|
-
.addColumn(
|
|
37
|
-
.addColumn(
|
|
38
|
-
.addColumn(
|
|
39
|
-
.addColumn(
|
|
40
|
-
.addColumn(
|
|
41
|
-
.addColumn(
|
|
42
|
-
.addColumn(
|
|
43
|
-
.addColumn(
|
|
44
|
-
.addColumn(
|
|
45
|
-
.addColumn(
|
|
46
|
-
.addColumn(
|
|
47
|
-
.addColumn(
|
|
48
|
-
.addColumn(
|
|
49
|
-
.addColumn(
|
|
50
|
-
.addColumn(
|
|
51
|
-
.addColumn(
|
|
52
|
-
.addColumn(
|
|
53
|
-
.addColumn(
|
|
54
|
-
.addColumn(
|
|
55
|
-
.addColumn(
|
|
56
|
-
.addColumn(
|
|
57
|
-
.addColumn(
|
|
58
|
-
.addColumn(
|
|
59
|
-
.addColumn(
|
|
60
|
-
.addColumn(
|
|
17
|
+
.addColumn('action_id', 'text', (col) => col.unique().notNull())
|
|
18
|
+
.addColumn('idsite', 'text')
|
|
19
|
+
.addColumn('idvisit', 'text')
|
|
20
|
+
.addColumn('actions', 'text')
|
|
21
|
+
.addColumn('country', 'text')
|
|
22
|
+
.addColumn('region', 'text')
|
|
23
|
+
.addColumn('city', 'text')
|
|
24
|
+
.addColumn('operatingsystemname', 'text')
|
|
25
|
+
.addColumn('devicemodel', 'text')
|
|
26
|
+
.addColumn('devicebrand', 'text')
|
|
27
|
+
.addColumn('visitduration', 'text')
|
|
28
|
+
.addColumn('dayssincefirstvisit', 'text')
|
|
29
|
+
.addColumn('visitortype', 'text')
|
|
30
|
+
.addColumn('sitename', 'text')
|
|
31
|
+
.addColumn('userid', 'text')
|
|
32
|
+
.addColumn('serverdateprettyfirstaction', 'date')
|
|
33
|
+
.addColumn('action_type', 'text')
|
|
34
|
+
.addColumn('action_eventcategory', 'text')
|
|
35
|
+
.addColumn('action_eventaction', 'text')
|
|
36
|
+
.addColumn('action_eventname', 'text')
|
|
37
|
+
.addColumn('action_eventvalue', 'numeric')
|
|
38
|
+
.addColumn('action_timespent', 'text')
|
|
39
|
+
.addColumn('action_timestamp', 'timestamptz', (col) => col.defaultTo(sql `now()`))
|
|
40
|
+
.addColumn('usercustomproperties', 'json')
|
|
41
|
+
.addColumn('usercustomdimensions', 'json')
|
|
42
|
+
.addColumn('dimension1', 'text')
|
|
43
|
+
.addColumn('dimension2', 'text')
|
|
44
|
+
.addColumn('dimension3', 'text')
|
|
45
|
+
.addColumn('dimension4', 'text')
|
|
46
|
+
.addColumn('dimension5', 'text')
|
|
47
|
+
.addColumn('dimension6', 'text')
|
|
48
|
+
.addColumn('dimension7', 'text')
|
|
49
|
+
.addColumn('dimension8', 'text')
|
|
50
|
+
.addColumn('dimension9', 'text')
|
|
51
|
+
.addColumn('dimension10', 'text')
|
|
52
|
+
.addColumn('action_url', 'text')
|
|
53
|
+
.addColumn('sitesearchkeyword', 'text')
|
|
54
|
+
.addColumn('action_title', 'text')
|
|
55
|
+
.addColumn('visitorid', 'text')
|
|
56
|
+
.addColumn('referrertype', 'text')
|
|
57
|
+
.addColumn('referrername', 'text')
|
|
61
58
|
.execute();
|
|
62
59
|
});
|
|
63
60
|
}
|
|
64
|
-
|
|
65
|
-
function down(db) {
|
|
61
|
+
export function down(db) {
|
|
66
62
|
return __awaiter(this, void 0, void 0, function* () {
|
|
67
|
-
yield db.schema.dropTable(
|
|
63
|
+
yield db.schema.dropTable(MATOMO_TABLE_NAME).execute();
|
|
68
64
|
});
|
|
69
65
|
}
|
|
70
|
-
exports.down = down;
|