@socialgouv/matomo-postgres 1.5.2-beta.4 → 1.6.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
process.env.MATOMO_SITE = "42";
|
|
16
|
+
process.env.PROJECT_NAME = "some-project";
|
|
17
|
+
process.env.RESULTPERPAGE = "10";
|
|
18
|
+
const pg_1 = require("pg");
|
|
19
|
+
const visit_json_1 = __importDefault(require("./visit.json"));
|
|
20
|
+
const importDate_1 = require("../importDate");
|
|
21
|
+
const TEST_DATE = new Date(2023, 3, 15);
|
|
22
|
+
let queries = [];
|
|
23
|
+
let result = {
|
|
24
|
+
command: "string",
|
|
25
|
+
rowCount: 0,
|
|
26
|
+
};
|
|
27
|
+
jest.mock("pg", () => {
|
|
28
|
+
const client = {
|
|
29
|
+
query: (query, values) => {
|
|
30
|
+
queries.push([query, values]);
|
|
31
|
+
return result;
|
|
32
|
+
},
|
|
33
|
+
release: jest.fn(),
|
|
34
|
+
};
|
|
35
|
+
const methods = {
|
|
36
|
+
connect: () => client,
|
|
37
|
+
on: jest.fn(),
|
|
38
|
+
query: jest.fn(),
|
|
39
|
+
};
|
|
40
|
+
return { Pool: jest.fn(() => methods) };
|
|
41
|
+
});
|
|
42
|
+
let pool;
|
|
43
|
+
beforeEach(() => {
|
|
44
|
+
pool = new pg_1.Pool();
|
|
45
|
+
queries = [];
|
|
46
|
+
});
|
|
47
|
+
afterEach(() => {
|
|
48
|
+
jest.clearAllMocks();
|
|
49
|
+
});
|
|
50
|
+
test("importDate: should import given date", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
51
|
+
const piwikApi = jest.fn();
|
|
52
|
+
piwikApi.mockImplementation((options, cb) => {
|
|
53
|
+
cb(null, [
|
|
54
|
+
Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 123 }),
|
|
55
|
+
Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 124 }),
|
|
56
|
+
]);
|
|
57
|
+
});
|
|
58
|
+
pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
|
|
59
|
+
yield (0, importDate_1.importDate)(piwikApi, TEST_DATE);
|
|
60
|
+
expect(piwikApi.mock.calls.length).toEqual(1);
|
|
61
|
+
expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
|
|
62
|
+
{
|
|
63
|
+
"date": "2023-04-15",
|
|
64
|
+
"filter_limit": 10,
|
|
65
|
+
"filter_offset": 0,
|
|
66
|
+
"filter_sort_order": "asc",
|
|
67
|
+
"idSite": "42",
|
|
68
|
+
"method": "Live.getLastVisitsDetails",
|
|
69
|
+
"period": "day",
|
|
70
|
+
}
|
|
71
|
+
`);
|
|
72
|
+
expect(queries[0]).toMatchInlineSnapshot(`
|
|
73
|
+
[
|
|
74
|
+
"select count(distinct "idvisit") as "count" from "matomo" where date(timezone('UTC', action_timestamp)) = $1",
|
|
75
|
+
[
|
|
76
|
+
"2023-04-15",
|
|
77
|
+
],
|
|
78
|
+
]
|
|
79
|
+
`);
|
|
80
|
+
expect(queries.length).toEqual(1 + visit_json_1.default.actionDetails.length * 2);
|
|
81
|
+
}));
|
|
82
|
+
test("importDate: should paginate matomo API calls and produce 46 queries", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
83
|
+
const piwikApi = jest.fn();
|
|
84
|
+
let calls = 0;
|
|
85
|
+
piwikApi.mockImplementation((options, cb) => {
|
|
86
|
+
cb(null, Array.from({ length: calls ? 5 : 10 }, (k, v) => (Object.assign(Object.assign({}, visit_json_1.default), { idVisit: k }))));
|
|
87
|
+
calls++;
|
|
88
|
+
});
|
|
89
|
+
pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
|
|
90
|
+
yield (0, importDate_1.importDate)(piwikApi, TEST_DATE);
|
|
91
|
+
expect(piwikApi.mock.calls.length).toEqual(2);
|
|
92
|
+
expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
|
|
93
|
+
{
|
|
94
|
+
"date": "2023-04-15",
|
|
95
|
+
"filter_limit": 10,
|
|
96
|
+
"filter_offset": 0,
|
|
97
|
+
"filter_sort_order": "asc",
|
|
98
|
+
"idSite": "42",
|
|
99
|
+
"method": "Live.getLastVisitsDetails",
|
|
100
|
+
"period": "day",
|
|
101
|
+
}
|
|
102
|
+
`);
|
|
103
|
+
expect(piwikApi.mock.calls[1][0]).toMatchInlineSnapshot(`
|
|
104
|
+
{
|
|
105
|
+
"date": "2023-04-15",
|
|
106
|
+
"filter_limit": 10,
|
|
107
|
+
"filter_offset": 10,
|
|
108
|
+
"filter_sort_order": "asc",
|
|
109
|
+
"idSite": "42",
|
|
110
|
+
"method": "Live.getLastVisitsDetails",
|
|
111
|
+
"period": "day",
|
|
112
|
+
}
|
|
113
|
+
`);
|
|
114
|
+
expect(queries.length).toEqual(1 + visit_json_1.default.actionDetails.length * 15);
|
|
115
|
+
expect(queries).toMatchSnapshot();
|
|
116
|
+
}));
|
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
process.env.MATOMO_SITE = "42";
|
|
7
7
|
process.env.PROJECT_NAME = "some-project";
|
|
8
|
+
process.env.RESULTPERPAGE = "10";
|
|
8
9
|
const visit_json_1 = __importDefault(require("./visit.json"));
|
|
9
10
|
const importEvent_1 = require("../importEvent");
|
|
10
11
|
test("getEventsFromMatomoVisit: should merge action events", () => {
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
process.env.MATOMO_SITE = "42";
|
|
16
|
+
process.env.PROJECT_NAME = "some-project";
|
|
17
|
+
process.env.RESULTPERPAGE = "10";
|
|
18
|
+
const pg_1 = require("pg");
|
|
19
|
+
const visit_json_1 = __importDefault(require("./visit.json"));
|
|
20
|
+
const index_1 = __importDefault(require("../index"));
|
|
21
|
+
const TEST_DATE = new Date(2023, 3, 1);
|
|
22
|
+
let queries = [];
|
|
23
|
+
let result = {
|
|
24
|
+
command: "string",
|
|
25
|
+
rowCount: 0,
|
|
26
|
+
};
|
|
27
|
+
jest.mock("pg", () => {
|
|
28
|
+
const client = {
|
|
29
|
+
query: (query, values) => {
|
|
30
|
+
queries.push([query, values]);
|
|
31
|
+
return result;
|
|
32
|
+
},
|
|
33
|
+
release: jest.fn(),
|
|
34
|
+
};
|
|
35
|
+
const methods = {
|
|
36
|
+
connect: () => client,
|
|
37
|
+
on: jest.fn(),
|
|
38
|
+
query: jest.fn(),
|
|
39
|
+
};
|
|
40
|
+
return { Pool: jest.fn(() => methods) };
|
|
41
|
+
});
|
|
42
|
+
let pool;
|
|
43
|
+
beforeEach(() => {
|
|
44
|
+
pool = new pg_1.Pool();
|
|
45
|
+
queries = [];
|
|
46
|
+
piwikApiCalls = [];
|
|
47
|
+
});
|
|
48
|
+
afterEach(() => {
|
|
49
|
+
jest.clearAllMocks();
|
|
50
|
+
});
|
|
51
|
+
let piwikApiCalls = [];
|
|
52
|
+
jest.mock("piwik-client", () => {
|
|
53
|
+
const matomoVisits = [
|
|
54
|
+
Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 123 }),
|
|
55
|
+
Object.assign(Object.assign({}, visit_json_1.default), { idVisit: 124 }),
|
|
56
|
+
];
|
|
57
|
+
class PiwikMock {
|
|
58
|
+
constructor(options) {
|
|
59
|
+
this.options = options;
|
|
60
|
+
}
|
|
61
|
+
api(options, cb) {
|
|
62
|
+
piwikApiCalls.push(options);
|
|
63
|
+
cb(null, matomoVisits);
|
|
64
|
+
("");
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
return PiwikMock;
|
|
68
|
+
});
|
|
69
|
+
test("run: should fetch the latest 5 days on matomo", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
70
|
+
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
71
|
+
yield (0, index_1.default)();
|
|
72
|
+
expect(piwikApiCalls).toMatchSnapshot();
|
|
73
|
+
}));
|
|
74
|
+
test("run: should fetch the latest event date if no date provided", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
75
|
+
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
76
|
+
yield (0, index_1.default)();
|
|
77
|
+
expect(queries[0]).toMatchSnapshot();
|
|
78
|
+
}));
|
|
79
|
+
test("run: should run based on existing data if any", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
80
|
+
// ensure we use the latest entry in DB
|
|
81
|
+
expect(1).toEqual(1);
|
|
82
|
+
}));
|
|
83
|
+
test("run: should run SQL queries", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
84
|
+
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
85
|
+
yield (0, index_1.default)();
|
|
86
|
+
expect(queries).toMatchSnapshot();
|
|
87
|
+
expect(queries.length).toEqual(1 + 5 * (6 + 1));
|
|
88
|
+
}));
|
package/dist/migrate-latest.js
CHANGED
|
@@ -46,10 +46,6 @@ function migrateToLatest() {
|
|
|
46
46
|
//@ts-ignore
|
|
47
47
|
.where("extname", "=", "pg_partman")
|
|
48
48
|
.executeTakeFirst();
|
|
49
|
-
if (extension) {
|
|
50
|
-
console.error("pg_partman extension detected; Skip migrations");
|
|
51
|
-
return;
|
|
52
|
-
}
|
|
53
49
|
const migrator = new kysely_1.Migrator({
|
|
54
50
|
db: db_1.db,
|
|
55
51
|
provider: new kysely_1.FileMigrationProvider({
|
package/package.json
CHANGED
|
@@ -1,190 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
const mock_pgQuery = jest.fn();
|
|
3
|
-
const mock_matomoApi = jest.fn();
|
|
4
|
-
// import formatISO from "date-fns/formatISO";
|
|
5
|
-
// import addDays from "date-fns/addDays";
|
|
6
|
-
// import { INITIAL_OFFSET } from "../config";
|
|
7
|
-
process.env.MATOMO_SITE = "42";
|
|
8
|
-
process.env.PROJECT_NAME = "some-project";
|
|
9
|
-
// import matomoVisit from "./visit.json";
|
|
10
|
-
// import run from "../index";
|
|
11
|
-
// const NB_REQUEST_TO_INIT_DB = 3; // Number of query to init DB (createTable.js)
|
|
12
|
-
// const TEST_DATE = new Date();
|
|
13
|
-
// @ts-ignore
|
|
14
|
-
//const isoDate = (date) => formatISO(date, { representation: "date" });
|
|
15
|
-
// jest.mock("pg", () => {
|
|
16
|
-
// class Client {
|
|
17
|
-
// escapeIdentifier(name) {
|
|
18
|
-
// return name;
|
|
19
|
-
// }
|
|
20
|
-
// end() {}
|
|
21
|
-
// connect() {
|
|
22
|
-
// return Promise.resolve();
|
|
23
|
-
// }
|
|
24
|
-
// query(...args) {
|
|
25
|
-
// return Promise.resolve(mock_pgQuery(...args));
|
|
26
|
-
// }
|
|
27
|
-
// }
|
|
28
|
-
// return {
|
|
29
|
-
// Client,
|
|
30
|
-
// };
|
|
31
|
-
// });
|
|
32
|
-
// jest.mock("piwik-client", () => {
|
|
33
|
-
// class MatomoClient {
|
|
34
|
-
// api(...args: any[]) {
|
|
35
|
-
// return mock_matomoApi(...args);
|
|
36
|
-
// }
|
|
37
|
-
// }
|
|
38
|
-
// return MatomoClient;
|
|
39
|
-
// });
|
|
40
|
-
test("test", () => { });
|
|
41
|
-
/*
|
|
42
|
-
|
|
43
|
-
beforeEach(() => {
|
|
44
|
-
jest.resetAllMocks();
|
|
45
|
-
jest.resetModules();
|
|
46
|
-
process.env.STARTDATE = "";
|
|
47
|
-
//process.env.OFFSET = "";
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
test("run: should create table", async () => {
|
|
51
|
-
mock_pgQuery.mockReturnValue({ rows: [] });
|
|
52
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
53
|
-
return cb(null, []);
|
|
54
|
-
});
|
|
55
|
-
await run();
|
|
56
|
-
expect(mock_pgQuery.mock.calls[0]).toMatchSnapshot();
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
test("run: should fetch the latest event date if no date provided", async () => {
|
|
60
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
61
|
-
mock_pgQuery.mockReturnValue({ rows: [] });
|
|
62
|
-
|
|
63
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
64
|
-
return cb(null, [
|
|
65
|
-
{
|
|
66
|
-
...matomoVisit,
|
|
67
|
-
idVisit: 123,
|
|
68
|
-
},
|
|
69
|
-
{
|
|
70
|
-
...matomoVisit,
|
|
71
|
-
idVisit: 124,
|
|
72
|
-
},
|
|
73
|
-
]);
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
await run();
|
|
77
|
-
|
|
78
|
-
// check matomo requests
|
|
79
|
-
expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
|
|
80
|
-
expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
|
|
81
|
-
|
|
82
|
-
// check db queries
|
|
83
|
-
expect(mock_pgQuery.mock.calls[NB_REQUEST_TO_INIT_DB][0]).toEqual(
|
|
84
|
-
"select action_timestamp from matomo order by action_timestamp desc limit 1"
|
|
85
|
-
);
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test("run: should resume using latest event date - offset if no date provided", async () => {
|
|
89
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
90
|
-
|
|
91
|
-
const LAST_EVENT_DATE_OFFSET = 2;
|
|
92
|
-
// @ts-ignore
|
|
93
|
-
const LAST_EVENT_DATE = addDays(TEST_DATE, -LAST_EVENT_DATE_OFFSET);
|
|
94
|
-
|
|
95
|
-
mock_pgQuery.mockReturnValue({ rows: [{ action_timestamp: LAST_EVENT_DATE.getTime() }] });
|
|
96
|
-
|
|
97
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
98
|
-
return cb(null, [
|
|
99
|
-
{
|
|
100
|
-
...matomoVisit,
|
|
101
|
-
idVisit: 123,
|
|
102
|
-
},
|
|
103
|
-
{
|
|
104
|
-
...matomoVisit,
|
|
105
|
-
idVisit: 124,
|
|
106
|
-
},
|
|
107
|
-
]);
|
|
108
|
-
});
|
|
109
|
-
|
|
110
|
-
await run();
|
|
111
|
-
|
|
112
|
-
// check matomo requests
|
|
113
|
-
expect(mock_matomoApi.mock.calls[0][0].date).toEqual(
|
|
114
|
-
// @ts-ignore
|
|
115
|
-
isoDate(addDays(LAST_EVENT_DATE, -parseInt(INITIAL_OFFSET)))
|
|
116
|
-
);
|
|
117
|
-
expect(mock_matomoApi.mock.calls[0][0].filter_offset).toEqual(0);
|
|
118
|
-
|
|
119
|
-
const daysCount = LAST_EVENT_DATE_OFFSET + parseInt(INITIAL_OFFSET) + 1;
|
|
120
|
-
expect(mock_matomoApi.mock.calls.length).toEqual(daysCount);
|
|
121
|
-
|
|
122
|
-
// check db queries
|
|
123
|
-
expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + daysCount * 7); // NB_REQUEST_TO_INIT_DB + select queries + days offset
|
|
124
|
-
});
|
|
125
|
-
|
|
126
|
-
test("run: should use today date if nothing in DB", async () => {
|
|
127
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
128
|
-
mock_pgQuery.mockReturnValue({ rows: [] });
|
|
129
|
-
|
|
130
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
131
|
-
return cb(null, [
|
|
132
|
-
{
|
|
133
|
-
...matomoVisit,
|
|
134
|
-
idVisit: 123,
|
|
135
|
-
},
|
|
136
|
-
]);
|
|
137
|
-
});
|
|
138
|
-
|
|
139
|
-
await run();
|
|
140
|
-
|
|
141
|
-
// check matomo requests
|
|
142
|
-
expect(mock_matomoApi.mock.calls.length).toEqual(1);
|
|
143
|
-
expect(mock_matomoApi.mock.calls[0][0].date).toEqual(isoDate(TEST_DATE));
|
|
144
|
-
|
|
145
|
-
// check the 4 events inserted
|
|
146
|
-
expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 5); // NB_REQUEST_TO_INIT_DB + check date + latest + 2 inserts
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
test("run: should use given date if any", async () => {
|
|
150
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
151
|
-
mock_pgQuery.mockReturnValue({ rows: [] });
|
|
152
|
-
|
|
153
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
154
|
-
return cb(null, [
|
|
155
|
-
{
|
|
156
|
-
...matomoVisit,
|
|
157
|
-
idVisit: 123,
|
|
158
|
-
},
|
|
159
|
-
]);
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
// @ts-ignore
|
|
163
|
-
await run(isoDate(addDays(TEST_DATE, -10)) + "T00:00:00.000Z");
|
|
164
|
-
|
|
165
|
-
expect(mock_matomoApi.mock.calls.length).toEqual(11);
|
|
166
|
-
expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 11 * 4); // NB_REQUEST_TO_INIT_DB + inserts. no initial select as date is provided
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
test("run: should use STARTDATE if any", async () => {
|
|
170
|
-
// @ts-ignore
|
|
171
|
-
process.env.STARTDATE = isoDate(addDays(TEST_DATE, -5)) + "T00:00:00.000Z";
|
|
172
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
173
|
-
mock_pgQuery.mockReturnValue({ rows: [] });
|
|
174
|
-
|
|
175
|
-
mock_matomoApi.mockImplementation((options, cb) => {
|
|
176
|
-
return cb(null, [
|
|
177
|
-
{
|
|
178
|
-
...matomoVisit,
|
|
179
|
-
idVisit: 123,
|
|
180
|
-
},
|
|
181
|
-
]);
|
|
182
|
-
});
|
|
183
|
-
|
|
184
|
-
await run();
|
|
185
|
-
|
|
186
|
-
expect(mock_matomoApi.mock.calls.length).toEqual(6);
|
|
187
|
-
|
|
188
|
-
expect(mock_pgQuery.mock.calls.length).toEqual(NB_REQUEST_TO_INIT_DB + 1 + 6 * 4); // NB_REQUEST_TO_INIT_DB + initial select + inserts.
|
|
189
|
-
});
|
|
190
|
-
*/
|