@socialgouv/matomo-postgres 2.3.2 → 2.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/bin/index.js +0 -9
- package/package.json +10 -4
- package/dist/PiwikClient.js +0 -95
- package/dist/__tests__/importDate.test.js +0 -123
- package/dist/__tests__/importEvent.test.js +0 -9
- package/dist/__tests__/migration-imports.test.js +0 -38
- package/dist/__tests__/run.test.js +0 -91
- package/dist/__tests__/visit.json +0 -104
- package/dist/config.js +0 -20
- package/dist/db.js +0 -44
- package/dist/importDate.js +0 -93
- package/dist/importEvent.js +0 -214
- package/dist/index.js +0 -106
- package/dist/migrate-down.js +0 -41
- package/dist/migrate-latest.js +0 -67
- package/dist/migrations/20230301-01-initial.js +0 -65
- package/dist/migrations/20230301-02-indexes.js +0 -100
- package/dist/migrations/20250425-01-add-resolution.js +0 -38
- package/dist/migrations/20250715-01-weekly-partitioning.js +0 -359
- package/dist/migrations/20250908-01-convention-analysis-index.js +0 -29
package/README.md
CHANGED
|
@@ -28,7 +28,7 @@ npx @socialgouv/matomo-postgres
|
|
|
28
28
|
```bash
|
|
29
29
|
npm install @socialgouv/matomo-postgres
|
|
30
30
|
# or
|
|
31
|
-
|
|
31
|
+
pnpm add @socialgouv/matomo-postgres
|
|
32
32
|
```
|
|
33
33
|
|
|
34
34
|
## ⚙️ Configuration
|
|
@@ -163,36 +163,36 @@ The tool creates a comprehensive table structure capturing:
|
|
|
163
163
|
3. **Run the Application**:
|
|
164
164
|
|
|
165
165
|
```bash
|
|
166
|
-
|
|
166
|
+
pnpm start
|
|
167
167
|
```
|
|
168
168
|
|
|
169
169
|
### Development Commands
|
|
170
170
|
|
|
171
171
|
```bash
|
|
172
172
|
# Build TypeScript
|
|
173
|
-
|
|
173
|
+
pnpm build
|
|
174
174
|
|
|
175
175
|
# Run tests
|
|
176
|
-
|
|
176
|
+
pnpm test
|
|
177
177
|
|
|
178
178
|
# Update test snapshots
|
|
179
|
-
|
|
179
|
+
pnpm test -u
|
|
180
180
|
|
|
181
181
|
# Lint code
|
|
182
|
-
|
|
182
|
+
pnpm lint
|
|
183
183
|
|
|
184
184
|
# Fix linting issues
|
|
185
|
-
|
|
185
|
+
pnpm lint:fix
|
|
186
186
|
|
|
187
187
|
# Run database migrations
|
|
188
|
-
|
|
188
|
+
pnpm migrate
|
|
189
189
|
```
|
|
190
190
|
|
|
191
191
|
## 🗄️ Database Migrations
|
|
192
192
|
|
|
193
193
|
Database schema is managed through Kysely migrations located in `./src/migrations/`:
|
|
194
194
|
|
|
195
|
-
Migrations run automatically on each `
|
|
195
|
+
Migrations run automatically on each `pnpm start` to ensure schema compatibility.
|
|
196
196
|
|
|
197
197
|
## 📊 Data Flow
|
|
198
198
|
|
package/bin/index.js
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
// DIAGNOSTIC: Log environment state at entry point
|
|
4
|
-
console.log('🔍 [DIAGNOSTIC] bin/index.js starting...')
|
|
5
|
-
console.log('🔍 [DIAGNOSTIC] PGDATABASE env var:', process.env.PGDATABASE ? `SET (length: ${process.env.PGDATABASE.length})` : 'NOT SET OR EMPTY')
|
|
6
|
-
console.log('🔍 [DIAGNOSTIC] NODE_ENV:', process.env.NODE_ENV || 'NOT SET')
|
|
7
|
-
console.log('🔍 [DIAGNOSTIC] Current working directory:', process.cwd())
|
|
8
|
-
console.log('🔍 [DIAGNOSTIC] About to import db module...\n')
|
|
9
|
-
|
|
10
3
|
import { db } from '../dist/db.js'
|
|
11
4
|
import run from '../dist/index.js'
|
|
12
5
|
import { startMigration } from '../dist/migrate-latest.js'
|
|
13
6
|
|
|
14
|
-
console.log('🔍 [DIAGNOSTIC] Modules imported successfully\n')
|
|
15
|
-
|
|
16
7
|
async function start(date) {
|
|
17
8
|
console.log(`\nRunning migrations\n`)
|
|
18
9
|
await startMigration()
|
package/package.json
CHANGED
|
@@ -1,24 +1,30 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@socialgouv/matomo-postgres",
|
|
3
3
|
"description": "Extract visitor events from Matomo API and push to Postgres",
|
|
4
|
-
"version": "2.3.
|
|
4
|
+
"version": "2.3.11",
|
|
5
|
+
"packageManager": "pnpm@10.28.1",
|
|
5
6
|
"types": "types/index.d.ts",
|
|
6
7
|
"license": "Apache-2.0",
|
|
7
8
|
"main": "dist/index.js",
|
|
9
|
+
"repository": {
|
|
10
|
+
"url": "https://github.com/SocialGouv/matomo-postgres"
|
|
11
|
+
},
|
|
8
12
|
"preferGlobal": true,
|
|
9
13
|
"publishConfig": {
|
|
10
14
|
"access": "public"
|
|
11
15
|
},
|
|
12
16
|
"type": "module",
|
|
13
|
-
"bin":
|
|
17
|
+
"bin": {
|
|
18
|
+
"matomo-postgres": "bin/index.js"
|
|
19
|
+
},
|
|
14
20
|
"files": [
|
|
15
21
|
"bin",
|
|
16
22
|
"dist"
|
|
17
23
|
],
|
|
18
24
|
"scripts": {
|
|
19
|
-
"start": "
|
|
25
|
+
"start": "pnpm migrate && node ./bin/index.js",
|
|
20
26
|
"build": "tsc",
|
|
21
|
-
"prepublish": "
|
|
27
|
+
"prepublish": "pnpm build",
|
|
22
28
|
"migrate": "node ./dist/migrate-latest.js",
|
|
23
29
|
"test": "jest --verbose",
|
|
24
30
|
"lint": "eslint .",
|
package/dist/PiwikClient.js
DELETED
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
import * as http from 'http';
|
|
2
|
-
import * as https from 'https';
|
|
3
|
-
import * as querystring from 'querystring';
|
|
4
|
-
import * as url from 'url';
|
|
5
|
-
export default class PiwikClient {
|
|
6
|
-
constructor(baseURL, token) {
|
|
7
|
-
const parsedUrl = url.parse(baseURL, true);
|
|
8
|
-
this.settings = {
|
|
9
|
-
apihost: parsedUrl.hostname || '',
|
|
10
|
-
apipath: parsedUrl.pathname || ''
|
|
11
|
-
};
|
|
12
|
-
// Determine protocol and set http module
|
|
13
|
-
switch (parsedUrl.protocol) {
|
|
14
|
-
case 'http:':
|
|
15
|
-
this.http = http;
|
|
16
|
-
this.settings.apiport = parsedUrl.port
|
|
17
|
-
? parseInt(parsedUrl.port, 10)
|
|
18
|
-
: 80;
|
|
19
|
-
break;
|
|
20
|
-
case 'https:':
|
|
21
|
-
this.http = https;
|
|
22
|
-
this.settings.apiport = parsedUrl.port
|
|
23
|
-
? parseInt(parsedUrl.port, 10)
|
|
24
|
-
: 443;
|
|
25
|
-
break;
|
|
26
|
-
default:
|
|
27
|
-
this.http = http;
|
|
28
|
-
this.settings.apiport = 80;
|
|
29
|
-
}
|
|
30
|
-
// Set token from URL query or constructor parameter
|
|
31
|
-
if (parsedUrl.query && parsedUrl.query.token_auth) {
|
|
32
|
-
this.settings.token = parsedUrl.query.token_auth;
|
|
33
|
-
}
|
|
34
|
-
if (token) {
|
|
35
|
-
this.settings.token = token;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
api(vars, cb) {
|
|
39
|
-
if (typeof vars !== 'object') {
|
|
40
|
-
vars = {};
|
|
41
|
-
}
|
|
42
|
-
// Set default values
|
|
43
|
-
vars.module = 'API';
|
|
44
|
-
vars.format = 'JSON';
|
|
45
|
-
// Set token if not provided in vars
|
|
46
|
-
if (vars.token_auth == null) {
|
|
47
|
-
vars.token_auth = this.settings.token;
|
|
48
|
-
}
|
|
49
|
-
// Extract token_auth for POST body
|
|
50
|
-
const token_auth = vars.token_auth;
|
|
51
|
-
const postData = querystring.stringify({ token_auth });
|
|
52
|
-
// Remove token_auth from URL query params
|
|
53
|
-
const queryVars = Object.assign({}, vars);
|
|
54
|
-
delete queryVars.token_auth;
|
|
55
|
-
// Prepare request options
|
|
56
|
-
const options = {
|
|
57
|
-
host: this.settings.apihost,
|
|
58
|
-
port: this.settings.apiport,
|
|
59
|
-
path: this.settings.apipath + '?' + querystring.stringify(queryVars),
|
|
60
|
-
method: 'POST',
|
|
61
|
-
headers: {
|
|
62
|
-
'Content-Type': 'application/x-www-form-urlencoded'
|
|
63
|
-
}
|
|
64
|
-
};
|
|
65
|
-
// Make HTTP POST request
|
|
66
|
-
const req = this.http.request(options, (response) => {
|
|
67
|
-
let data = '';
|
|
68
|
-
// Collect data chunks
|
|
69
|
-
response.on('data', (chunk) => {
|
|
70
|
-
data += chunk;
|
|
71
|
-
});
|
|
72
|
-
// Process complete response
|
|
73
|
-
response.on('end', () => {
|
|
74
|
-
try {
|
|
75
|
-
const resObj = JSON.parse(data);
|
|
76
|
-
if (resObj.result === 'error') {
|
|
77
|
-
return cb(new Error(resObj.message), null);
|
|
78
|
-
}
|
|
79
|
-
return cb(null, resObj);
|
|
80
|
-
}
|
|
81
|
-
catch (error) {
|
|
82
|
-
return cb(error instanceof Error ? error : new Error(String(error)), null);
|
|
83
|
-
}
|
|
84
|
-
});
|
|
85
|
-
});
|
|
86
|
-
// Handle request errors
|
|
87
|
-
req.on('error', (error) => {
|
|
88
|
-
cb(error, null);
|
|
89
|
-
});
|
|
90
|
-
// Write POST data and end request
|
|
91
|
-
req.write(postData);
|
|
92
|
-
req.end();
|
|
93
|
-
return req;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
@@ -1,123 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
// Set environment variables BEFORE any imports
|
|
11
|
-
process.env.MATOMO_SITE = '42';
|
|
12
|
-
process.env.PROJECT_NAME = 'some-project';
|
|
13
|
-
process.env.RESULTPERPAGE = '10';
|
|
14
|
-
process.env.DESTINATION_TABLE = 'matomo';
|
|
15
|
-
import { Pool } from 'pg';
|
|
16
|
-
import { importDate } from '../importDate';
|
|
17
|
-
import matomoVisit from './visit.json';
|
|
18
|
-
const TEST_DATE = new Date(2023, 3, 15);
|
|
19
|
-
let queries = [];
|
|
20
|
-
const result = {
|
|
21
|
-
command: 'string',
|
|
22
|
-
rowCount: 0
|
|
23
|
-
};
|
|
24
|
-
jest.mock('pg', () => {
|
|
25
|
-
const client = {
|
|
26
|
-
query: (query, values) => {
|
|
27
|
-
queries.push([query, values]);
|
|
28
|
-
return result;
|
|
29
|
-
},
|
|
30
|
-
release: jest.fn()
|
|
31
|
-
};
|
|
32
|
-
const methods = {
|
|
33
|
-
connect: () => client,
|
|
34
|
-
on: jest.fn(),
|
|
35
|
-
query: jest.fn()
|
|
36
|
-
};
|
|
37
|
-
return { Pool: jest.fn(() => methods) };
|
|
38
|
-
});
|
|
39
|
-
let pool;
|
|
40
|
-
beforeEach(() => {
|
|
41
|
-
pool = new Pool();
|
|
42
|
-
queries = [];
|
|
43
|
-
});
|
|
44
|
-
afterEach(() => {
|
|
45
|
-
jest.clearAllMocks();
|
|
46
|
-
});
|
|
47
|
-
test('importDate: should import given date', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
48
|
-
const piwikApi = jest.fn();
|
|
49
|
-
piwikApi.mockImplementation((options, cb) => {
|
|
50
|
-
cb(null, [
|
|
51
|
-
Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
|
|
52
|
-
Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
|
|
53
|
-
]);
|
|
54
|
-
});
|
|
55
|
-
pool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
|
|
56
|
-
yield importDate(piwikApi, TEST_DATE);
|
|
57
|
-
expect(piwikApi.mock.calls.length).toEqual(1);
|
|
58
|
-
expect(piwikApi.mock.calls[0][0]).toMatchInlineSnapshot(`
|
|
59
|
-
{
|
|
60
|
-
"date": "2023-04-15",
|
|
61
|
-
"filter_limit": 10,
|
|
62
|
-
"filter_offset": 0,
|
|
63
|
-
"filter_sort_order": "asc",
|
|
64
|
-
"idSite": "42",
|
|
65
|
-
"method": "Live.getLastVisitsDetails",
|
|
66
|
-
"period": "day",
|
|
67
|
-
}
|
|
68
|
-
`);
|
|
69
|
-
expect(queries[0]).toMatchInlineSnapshot(`
|
|
70
|
-
[
|
|
71
|
-
"select count(distinct "idvisit") as "count" from "matomo" where date(timezone('UTC', action_timestamp)) = $1",
|
|
72
|
-
[
|
|
73
|
-
"2023-04-15",
|
|
74
|
-
],
|
|
75
|
-
]
|
|
76
|
-
`);
|
|
77
|
-
expect(queries.length).toEqual(1 + matomoVisit.actionDetails.length * 2);
|
|
78
|
-
}));
|
|
79
|
-
test('importDate: should handle pagination across multiple pages', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
80
|
-
const piwikApi = jest.fn();
|
|
81
|
-
// Mock first call to return exactly 10 visits (triggers pagination)
|
|
82
|
-
piwikApi
|
|
83
|
-
.mockImplementationOnce((options, cb) => {
|
|
84
|
-
const visits = Array.from({ length: 10 }, (_, i) => (Object.assign(Object.assign({}, matomoVisit), { idVisit: 200 + i })));
|
|
85
|
-
cb(null, visits);
|
|
86
|
-
})
|
|
87
|
-
// Mock second call to return 5 visits (stops pagination)
|
|
88
|
-
.mockImplementationOnce((options, cb) => {
|
|
89
|
-
const visits = Array.from({ length: 5 }, (_, i) => (Object.assign(Object.assign({}, matomoVisit), { idVisit: 300 + i })));
|
|
90
|
-
cb(null, visits);
|
|
91
|
-
});
|
|
92
|
-
// Mock database query for record count
|
|
93
|
-
pool.query.mockResolvedValue({ rows: [], rowCount: 0 });
|
|
94
|
-
const result = yield importDate(piwikApi, TEST_DATE);
|
|
95
|
-
// Should make exactly 2 API calls due to pagination
|
|
96
|
-
expect(piwikApi.mock.calls.length).toEqual(2);
|
|
97
|
-
// First call should have offset 0
|
|
98
|
-
expect(piwikApi.mock.calls[0][0]).toMatchObject({
|
|
99
|
-
date: '2023-04-15',
|
|
100
|
-
filter_limit: 10,
|
|
101
|
-
filter_offset: 0,
|
|
102
|
-
filter_sort_order: 'asc',
|
|
103
|
-
idSite: '42',
|
|
104
|
-
method: 'Live.getLastVisitsDetails',
|
|
105
|
-
period: 'day'
|
|
106
|
-
});
|
|
107
|
-
// Second call should have offset 10
|
|
108
|
-
expect(piwikApi.mock.calls[1][0]).toMatchObject({
|
|
109
|
-
date: '2023-04-15',
|
|
110
|
-
filter_limit: 10,
|
|
111
|
-
filter_offset: 10,
|
|
112
|
-
filter_sort_order: 'asc',
|
|
113
|
-
idSite: '42',
|
|
114
|
-
method: 'Live.getLastVisitsDetails',
|
|
115
|
-
period: 'day'
|
|
116
|
-
});
|
|
117
|
-
// Should process all events from both pages
|
|
118
|
-
// 15 visits total × 3 actionDetails each = 45 events
|
|
119
|
-
expect(result.length).toEqual(45);
|
|
120
|
-
// Verify database queries: 1 count query + (45 events × 1 query per event)
|
|
121
|
-
// Note: Each event generates 1 database query for insertion
|
|
122
|
-
expect(queries.length).toEqual(1 + 45);
|
|
123
|
-
}));
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
import { getEventsFromMatomoVisit } from '../importEvent.js';
|
|
2
|
-
import matomoVisit from './visit.json';
|
|
3
|
-
process.env.MATOMO_SITE = '42';
|
|
4
|
-
process.env.PROJECT_NAME = 'some-project';
|
|
5
|
-
process.env.RESULTPERPAGE = '10';
|
|
6
|
-
test('getEventsFromMatomoVisit: should merge action events', () => {
|
|
7
|
-
const visits = getEventsFromMatomoVisit(matomoVisit);
|
|
8
|
-
expect(visits).toMatchSnapshot();
|
|
9
|
-
});
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
import { readdir, readFile } from 'fs/promises';
|
|
11
|
-
import { join } from 'path';
|
|
12
|
-
describe('Migration Import Validation', () => {
|
|
13
|
-
it('should not have src/ imports in migration files', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
14
|
-
const migrationsDir = join(__dirname, '../migrations');
|
|
15
|
-
const migrationFiles = yield readdir(migrationsDir);
|
|
16
|
-
const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
|
|
17
|
-
const problematicFiles = [];
|
|
18
|
-
for (const file of tsFiles) {
|
|
19
|
-
const filePath = join(migrationsDir, file);
|
|
20
|
-
const content = yield readFile(filePath, 'utf-8');
|
|
21
|
-
// Check for imports from 'src/' paths
|
|
22
|
-
const srcImportRegex = /(?:import\s+[^;]+from\s+['"]src\/|require\s*\(\s*['"]src\/)/g;
|
|
23
|
-
const matches = content.match(srcImportRegex);
|
|
24
|
-
if (matches) {
|
|
25
|
-
problematicFiles.push(`${file}: ${matches.join(', ')}`);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
if (problematicFiles.length > 0) {
|
|
29
|
-
fail(`Migration files contain problematic src/ imports that will fail in production:\n${problematicFiles.join('\n')}\n\nMigration files should use direct constants or relative imports instead.`);
|
|
30
|
-
}
|
|
31
|
-
}));
|
|
32
|
-
it('should have at least one migration file', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
33
|
-
const migrationsDir = join(__dirname, '../migrations');
|
|
34
|
-
const migrationFiles = yield readdir(migrationsDir);
|
|
35
|
-
const tsFiles = migrationFiles.filter((file) => file.endsWith('.ts'));
|
|
36
|
-
expect(tsFiles.length).toBeGreaterThan(0);
|
|
37
|
-
}));
|
|
38
|
-
});
|
|
@@ -1,91 +0,0 @@
|
|
|
1
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
-
});
|
|
9
|
-
};
|
|
10
|
-
process.env.MATOMO_SITE = '42';
|
|
11
|
-
process.env.PROJECT_NAME = 'some-project';
|
|
12
|
-
process.env.RESULTPERPAGE = '10';
|
|
13
|
-
delete process.env.INITIAL_OFFSET;
|
|
14
|
-
delete process.env.DESTINATION_TABLE;
|
|
15
|
-
delete process.env.STARTDATE;
|
|
16
|
-
// Clear STARTDATE to avoid conflicts with fake timers
|
|
17
|
-
const TEST_DATE = new Date(2023, 3, 1);
|
|
18
|
-
let queries = [];
|
|
19
|
-
let piwikApiCalls = [];
|
|
20
|
-
const result = {
|
|
21
|
-
command: 'string',
|
|
22
|
-
rowCount: 0
|
|
23
|
-
};
|
|
24
|
-
jest.mock('pg', () => {
|
|
25
|
-
const client = {
|
|
26
|
-
query: (query, values) => {
|
|
27
|
-
queries.push([query, values]);
|
|
28
|
-
return result;
|
|
29
|
-
},
|
|
30
|
-
release: jest.fn()
|
|
31
|
-
};
|
|
32
|
-
const methods = {
|
|
33
|
-
connect: () => client,
|
|
34
|
-
on: jest.fn(),
|
|
35
|
-
query: jest.fn()
|
|
36
|
-
};
|
|
37
|
-
return { Pool: jest.fn(() => methods) };
|
|
38
|
-
});
|
|
39
|
-
jest.mock('../PiwikClient', () => {
|
|
40
|
-
class PiwikMock {
|
|
41
|
-
constructor(options) {
|
|
42
|
-
this.options = options;
|
|
43
|
-
}
|
|
44
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type
|
|
45
|
-
api(options, cb) {
|
|
46
|
-
return __awaiter(this, void 0, void 0, function* () {
|
|
47
|
-
// Import the visit data dynamically to avoid circular dependency
|
|
48
|
-
const { default: matomoVisit } = yield import('./visit.json');
|
|
49
|
-
const matomoVisits = [
|
|
50
|
-
Object.assign(Object.assign({}, matomoVisit), { idVisit: 123 }),
|
|
51
|
-
Object.assign(Object.assign({}, matomoVisit), { idVisit: 124 })
|
|
52
|
-
];
|
|
53
|
-
piwikApiCalls.push(options);
|
|
54
|
-
cb(null, matomoVisits);
|
|
55
|
-
});
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
return PiwikMock;
|
|
59
|
-
});
|
|
60
|
-
// Import after mocks are set up
|
|
61
|
-
import run from '../index';
|
|
62
|
-
beforeEach(() => {
|
|
63
|
-
queries = [];
|
|
64
|
-
piwikApiCalls = [];
|
|
65
|
-
});
|
|
66
|
-
afterEach(() => {
|
|
67
|
-
jest.clearAllMocks();
|
|
68
|
-
});
|
|
69
|
-
test('run: should fetch the latest 5 days on matomo', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
70
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
71
|
-
yield run();
|
|
72
|
-
expect(piwikApiCalls).toMatchSnapshot();
|
|
73
|
-
}));
|
|
74
|
-
test('run: should fetch the latest event date if no date provided', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
75
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
76
|
-
yield run();
|
|
77
|
-
expect(queries[0]).toMatchSnapshot();
|
|
78
|
-
}));
|
|
79
|
-
test('run: should run based on existing data if any', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
80
|
-
// ensure we use the latest entry in DB
|
|
81
|
-
expect(1).toEqual(1);
|
|
82
|
-
}));
|
|
83
|
-
test('run: should run SQL queries', () => __awaiter(void 0, void 0, void 0, function* () {
|
|
84
|
-
jest.useFakeTimers().setSystemTime(TEST_DATE.getTime());
|
|
85
|
-
yield run();
|
|
86
|
-
expect(queries).toMatchSnapshot();
|
|
87
|
-
// Updated expectation based on actual behavior with INITIAL_OFFSET=3 (5 days total: 3 days before + today + 1 day after)
|
|
88
|
-
// 5 days * (6 events per day + 1 count query per day)
|
|
89
|
-
// Note: The initial findLastEventInMatomo query is not captured in the test mock environment
|
|
90
|
-
expect(queries.length).toEqual(5 * (6 + 1));
|
|
91
|
-
}));
|
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"idVisit": "124",
|
|
3
|
-
"idSite": "42",
|
|
4
|
-
"country": "Argentine",
|
|
5
|
-
"operatingSystemName": "Mac",
|
|
6
|
-
"deviceBrand": "Inconnu",
|
|
7
|
-
"deviceModel": "Générique Bureau",
|
|
8
|
-
"visitDuration": "300",
|
|
9
|
-
"daysSinceFirstVisit": "23",
|
|
10
|
-
"actions": "2",
|
|
11
|
-
"visitorType": "returningCustomer",
|
|
12
|
-
"visitorId": "visitorId",
|
|
13
|
-
"referrerType": "referrerType",
|
|
14
|
-
"referrerName": "referrerName",
|
|
15
|
-
"siteName": "tests",
|
|
16
|
-
"userId": "24",
|
|
17
|
-
"region": "Buenos Aires",
|
|
18
|
-
"city": "Buenos Aires",
|
|
19
|
-
"resolution": "1920x1080",
|
|
20
|
-
"dimension1": "guest",
|
|
21
|
-
"dimension3": "page",
|
|
22
|
-
"dimension6": "shop",
|
|
23
|
-
"dimension7": "v1.2.3",
|
|
24
|
-
"dimension8": "fr",
|
|
25
|
-
"dimension9": "light",
|
|
26
|
-
"dimension10": "36",
|
|
27
|
-
"firstActionTimestamp": 1629496512,
|
|
28
|
-
"actionDetails": [
|
|
29
|
-
{
|
|
30
|
-
"type": "event",
|
|
31
|
-
"url": "https://dive-shop.net/products/basic-wetsuit/",
|
|
32
|
-
"pageIdAction": "304",
|
|
33
|
-
"idpageview": "",
|
|
34
|
-
"serverTimePretty": "20 août 2021 21:35:18",
|
|
35
|
-
"pageId": "19696671",
|
|
36
|
-
"eventCategory": "Ecommerce",
|
|
37
|
-
"eventAction": "Cart change",
|
|
38
|
-
"timeSpent": 48,
|
|
39
|
-
"timeSpentPretty": "48s",
|
|
40
|
-
"pageviewPosition": "8",
|
|
41
|
-
"timestamp": 1629495318,
|
|
42
|
-
"icon": "plugins/Morpheus/images/event.png",
|
|
43
|
-
"iconSVG": "plugins/Morpheus/images/event.svg",
|
|
44
|
-
"title": "Evènement",
|
|
45
|
-
"subtitle": "Catégorie: \"Ecommerce', Action: \"Cart change\"",
|
|
46
|
-
"eventName": "added - Basic Wetsuit",
|
|
47
|
-
"eventValue": 1,
|
|
48
|
-
"dimension2": "julien",
|
|
49
|
-
"dimension4": "indonesia",
|
|
50
|
-
"dimension5": "diving",
|
|
51
|
-
"customVariables": {
|
|
52
|
-
"1": {
|
|
53
|
-
"customVariableName1": "page-author",
|
|
54
|
-
"customVariableValue1": "Julien"
|
|
55
|
-
},
|
|
56
|
-
"2": {
|
|
57
|
-
"customVariableName2": "post-age",
|
|
58
|
-
"customVariableValue2": "-430 days"
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
},
|
|
62
|
-
{
|
|
63
|
-
"type": "action",
|
|
64
|
-
"url": "https://dive-shop.net/products/diving-boots/",
|
|
65
|
-
"pageTitle": "Divezone Brand Diving Boots - Divezone Store",
|
|
66
|
-
"pageIdAction": "60",
|
|
67
|
-
"idpageview": "8CDIez",
|
|
68
|
-
"serverTimePretty": "20 août 2021 21:30:25",
|
|
69
|
-
"pageId": "19696664",
|
|
70
|
-
"timeSpent": "2",
|
|
71
|
-
"timeSpentPretty": "2s",
|
|
72
|
-
"pageviewPosition": "5",
|
|
73
|
-
"title": "Divezone Brand Diving Boots - Divezone Store",
|
|
74
|
-
"subtitle": "https://dive-shop.net/products/diving-boots/",
|
|
75
|
-
"icon": "",
|
|
76
|
-
"iconSVG": "plugins/Morpheus/images/action.svg",
|
|
77
|
-
"timestamp": 1629495025,
|
|
78
|
-
"dimension2": "julien",
|
|
79
|
-
"dimension4": "indonesia",
|
|
80
|
-
"dimension5": "diving"
|
|
81
|
-
},
|
|
82
|
-
{
|
|
83
|
-
"type": "search",
|
|
84
|
-
"url": "https://dive-shop.net/products/diving-boots/",
|
|
85
|
-
"pageTitle": "Divezone Brand Diving Boots - Divezone Store",
|
|
86
|
-
"pageIdAction": "60",
|
|
87
|
-
"idpageview": "8CDIez",
|
|
88
|
-
"serverTimePretty": "20 août 2021 21:30:25",
|
|
89
|
-
"pageId": "19696664",
|
|
90
|
-
"timeSpent": "2",
|
|
91
|
-
"timeSpentPretty": "2s",
|
|
92
|
-
"pageviewPosition": "5",
|
|
93
|
-
"title": "Divezone Brand Diving Boots - Divezone Store",
|
|
94
|
-
"subtitle": "https://dive-shop.net/products/diving-boots/",
|
|
95
|
-
"icon": "",
|
|
96
|
-
"iconSVG": "plugins/Morpheus/images/action.svg",
|
|
97
|
-
"timestamp": 1629495022,
|
|
98
|
-
"dimension2": "julien",
|
|
99
|
-
"dimension4": "indonesia",
|
|
100
|
-
"dimension5": "diving",
|
|
101
|
-
"siteSearchKeyword": "scuba"
|
|
102
|
-
}
|
|
103
|
-
]
|
|
104
|
-
}
|
package/dist/config.js
DELETED
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
// DIAGNOSTIC: Log when config is being loaded
|
|
2
|
-
console.log('🔍 [DIAGNOSTIC] config.ts module loading...');
|
|
3
|
-
console.log('🔍 [DIAGNOSTIC] Reading environment variables:');
|
|
4
|
-
console.log(' - PGDATABASE:', process.env.PGDATABASE ? `SET (length: ${process.env.PGDATABASE.length})` : 'NOT SET OR EMPTY');
|
|
5
|
-
console.log(' - MATOMO_URL:', process.env.MATOMO_URL ? 'SET' : 'NOT SET (will use default)');
|
|
6
|
-
console.log(' - MATOMO_SITE:', process.env.MATOMO_SITE ? 'SET' : 'NOT SET (will use default)');
|
|
7
|
-
console.log(' - MATOMO_KEY:', process.env.MATOMO_KEY ? 'SET' : 'NOT SET');
|
|
8
|
-
export const MATOMO_KEY = process.env.MATOMO_KEY || '';
|
|
9
|
-
export const MATOMO_URL = process.env.MATOMO_URL || 'https://matomo.fabrique.social.gouv.fr/';
|
|
10
|
-
export const MATOMO_SITE = process.env.MATOMO_SITE || 0;
|
|
11
|
-
export const PGDATABASE = process.env.PGDATABASE || '';
|
|
12
|
-
export const INITIAL_OFFSET = process.env.INITIAL_OFFSET || '3';
|
|
13
|
-
export const RESULTPERPAGE = process.env.RESULTPERPAGE || '500';
|
|
14
|
-
export const FORCE_STARTDATE = process.env.FORCE_STARTDATE === 'true';
|
|
15
|
-
console.log('🔍 [DIAGNOSTIC] config.ts module loaded\n');
|
|
16
|
-
// We will create both a normal and a partitioned table (MATOMO_TABLE_NAME and PARTITIONED_MATOMO_TABLE_NAME)
|
|
17
|
-
// and use DESTINATION_TABLE to determine which one to write to.
|
|
18
|
-
export const DESTINATION_TABLE = process.env.DESTINATION_TABLE || 'matomo';
|
|
19
|
-
export const MATOMO_TABLE_NAME = process.env.MATOMO_TABLE_NAME || 'matomo';
|
|
20
|
-
export const PARTITIONED_MATOMO_TABLE_NAME = process.env.PARTITIONED_MATOMO_TABLE_NAME || 'matomo_partitioned';
|
package/dist/db.js
DELETED
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import startDebug from 'debug';
|
|
2
|
-
import { Kysely, PostgresDialect } from 'kysely';
|
|
3
|
-
import pkg from 'pg';
|
|
4
|
-
const { Pool } = pkg;
|
|
5
|
-
import { PGDATABASE } from './config.js';
|
|
6
|
-
startDebug('db');
|
|
7
|
-
// DIAGNOSTIC: Log environment state at module load time
|
|
8
|
-
console.log('🔍 [DIAGNOSTIC] db.ts module loading...');
|
|
9
|
-
console.log('🔍 [DIAGNOSTIC] PGDATABASE value:', PGDATABASE ? `SET (length: ${PGDATABASE.length})` : 'NOT SET OR EMPTY');
|
|
10
|
-
console.log('🔍 [DIAGNOSTIC] Pool constructor type:', typeof Pool);
|
|
11
|
-
export const pool = new Pool({
|
|
12
|
-
connectionString: PGDATABASE,
|
|
13
|
-
ssl: {
|
|
14
|
-
rejectUnauthorized: false
|
|
15
|
-
}
|
|
16
|
-
});
|
|
17
|
-
// DIAGNOSTIC: Log pool creation details
|
|
18
|
-
console.log('🔍 [DIAGNOSTIC] Pool created:', pool ? 'YES' : 'NO');
|
|
19
|
-
console.log('🔍 [DIAGNOSTIC] Pool has connect method:', pool && typeof pool.connect === 'function' ? 'YES' : 'NO');
|
|
20
|
-
console.log('🔍 [DIAGNOSTIC] Pool object keys:', pool ? Object.keys(pool).join(', ') : 'N/A');
|
|
21
|
-
// Validate pool is properly initialized
|
|
22
|
-
if (!pool || typeof pool.connect !== 'function') {
|
|
23
|
-
throw new Error('Failed to initialize PostgreSQL connection pool');
|
|
24
|
-
}
|
|
25
|
-
// DIAGNOSTIC: Log dialect creation
|
|
26
|
-
console.log('🔍 [DIAGNOSTIC] Creating PostgresDialect with pool...');
|
|
27
|
-
const dialect = new PostgresDialect({ pool: pool });
|
|
28
|
-
console.log('🔍 [DIAGNOSTIC] PostgresDialect created:', dialect ? 'YES' : 'NO');
|
|
29
|
-
export const db = new Kysely({
|
|
30
|
-
dialect: dialect,
|
|
31
|
-
log(event) {
|
|
32
|
-
if (event.level === 'query') {
|
|
33
|
-
// debug(event.query.sql)
|
|
34
|
-
// debug(event.query.parameters)
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
});
|
|
38
|
-
// DIAGNOSTIC: Log final db instance
|
|
39
|
-
console.log('🔍 [DIAGNOSTIC] Kysely db instance created:', db ? 'YES' : 'NO');
|
|
40
|
-
console.log('🔍 [DIAGNOSTIC] db.ts module loaded successfully\n');
|
|
41
|
-
// Validate the Kysely instance
|
|
42
|
-
if (!db) {
|
|
43
|
-
throw new Error('Failed to initialize Kysely database instance');
|
|
44
|
-
}
|