@opentermsarchive/engine 7.2.4 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/env.js +4 -2
- package/bin/ota-dataset.js +2 -2
- package/config/default.json +1 -1
- package/package.json +48 -57
- package/scripts/dataset/export/index.test.js +2 -4
- package/scripts/dataset/publish/github/index.js +1 -1
- package/scripts/dataset/publish/gitlab/index.js +1 -1
- package/scripts/metadata/index.mocha.js +2 -2
- package/scripts/reporter/duplicate/index.js +4 -2
- package/src/archivist/extract/index.test.js +1 -2
- package/src/archivist/extract/markdown.js +6 -1
- package/src/archivist/fetcher/fullDomFetcher.test.js +2 -3
- package/src/archivist/fetcher/index.test.js +2 -3
- package/src/archivist/index.test.js +2 -3
- package/src/archivist/recorder/index.test.js +1 -3
- package/src/archivist/recorder/repositories/git/dataMapper.js +0 -2
- package/src/archivist/recorder/repositories/git/index.js +0 -2
- package/src/archivist/recorder/repositories/git/index.test.js +1 -3
- package/src/archivist/recorder/repositories/mongo/index.js +6 -2
- package/src/archivist/recorder/repositories/mongo/index.test.js +16 -5
- package/src/archivist/recorder/snapshot.test.js +1 -3
- package/src/archivist/recorder/version.test.js +1 -3
- package/src/archivist/services/index.test.js +2 -3
- package/src/archivist/services/service.test.js +1 -3
- package/src/archivist/services/sourceDocument.test.js +1 -3
- package/src/archivist/services/terms.test.js +1 -3
- package/src/collection-api/logger.js +1 -1
- package/src/collection-api/routes/metadata.js +2 -2
- package/src/collection-api/routes/metadata.test.js +21 -31
- package/src/collection-api/routes/versions.test.js +1 -2
- package/src/collection-api/server.js +0 -1
- package/src/index.js +2 -2
- package/src/reporter/github/index.js +1 -1
- package/src/reporter/labels.test.js +1 -3
package/bin/env.js
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import 'dotenv/config';
|
|
2
|
-
|
|
3
1
|
import path from 'path';
|
|
4
2
|
import { fileURLToPath } from 'url';
|
|
5
3
|
|
|
4
|
+
import dotenv from 'dotenv';
|
|
5
|
+
|
|
6
|
+
dotenv.config({ quiet: true });
|
|
7
|
+
|
|
6
8
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
7
9
|
|
|
8
10
|
process.env.SUPPRESS_NO_CONFIG_WARNING = 'y'; // Caller don't get "no config files" warnings if it does define configuration files
|
package/bin/ota-dataset.js
CHANGED
|
@@ -3,7 +3,7 @@ import './env.js';
|
|
|
3
3
|
|
|
4
4
|
import { program } from 'commander';
|
|
5
5
|
import config from 'config';
|
|
6
|
-
import
|
|
6
|
+
import { Cron } from 'croner';
|
|
7
7
|
import cronstrue from 'cronstrue';
|
|
8
8
|
|
|
9
9
|
import { release } from '../scripts/dataset/index.js';
|
|
@@ -34,5 +34,5 @@ if (!schedule) {
|
|
|
34
34
|
logger.info('The scheduler is running…');
|
|
35
35
|
logger.info(`Dataset will be published ${humanReadableSchedule.toLowerCase()} in the timezone of this machine`);
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
new Cron(config.get('@opentermsarchive/engine.dataset.publishingSchedule'), () => release(options)); // eslint-disable-line no-new
|
|
38
38
|
}
|
package/config/default.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opentermsarchive/engine",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "9.0.0",
|
|
4
4
|
"description": "Tracks and makes visible changes to the terms of online services",
|
|
5
5
|
"homepage": "https://opentermsarchive.org",
|
|
6
6
|
"bugs": {
|
|
@@ -30,7 +30,6 @@
|
|
|
30
30
|
".eslintrc.yaml"
|
|
31
31
|
],
|
|
32
32
|
"scripts": {
|
|
33
|
-
"changelog": "node scripts/changelog/index.js",
|
|
34
33
|
"commit-messages:lint": "commitlint --from=main --to=HEAD",
|
|
35
34
|
"dataset:generate": "node bin/ota.js dataset",
|
|
36
35
|
"dataset:release": "node bin/ota.js dataset --publish --remove-local-copy",
|
|
@@ -41,88 +40,80 @@
|
|
|
41
40
|
"lint": "eslint src test scripts bin",
|
|
42
41
|
"lint:fix": "npm run lint -- --fix",
|
|
43
42
|
"metadata:validate": "node bin/ota.js validate metadata",
|
|
44
|
-
"start": "node -r dotenv/config --max-http-header-size=32768 bin/ota.js track",
|
|
43
|
+
"start": "DOTENV_CONFIG_QUIET=true node -r dotenv/config --max-http-header-size=32768 bin/ota.js track",
|
|
45
44
|
"start:api": "node bin/ota.js serve",
|
|
46
45
|
"start:scheduler": "npm start -- --schedule",
|
|
47
46
|
"test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
|
|
48
|
-
"test:only": "cross-env NODE_ENV=test mocha --recursive",
|
|
49
47
|
"posttest": "npm run lint",
|
|
50
|
-
"test:debug": "npm run test -- --inspect-brk --exit"
|
|
48
|
+
"test:debug": "npm run test -- --inspect-brk --exit",
|
|
49
|
+
"test:only": "cross-env NODE_ENV=test mocha --recursive"
|
|
51
50
|
},
|
|
52
51
|
"dependencies": {
|
|
53
|
-
"@accordproject/
|
|
54
|
-
"@accordproject/
|
|
55
|
-
"@
|
|
56
|
-
"@
|
|
52
|
+
"@accordproject/concerto-cto": "^3.24.0",
|
|
53
|
+
"@accordproject/concerto-util": "^3.24.0",
|
|
54
|
+
"@accordproject/markdown-cicero": "^0.16.26",
|
|
55
|
+
"@accordproject/markdown-pdf": "^0.16.25",
|
|
57
56
|
"abort-controller": "^3.0.0",
|
|
58
57
|
"ajv": "^8.17.1",
|
|
59
58
|
"ajv-formats": "^3.0.1",
|
|
60
|
-
"archiver": "^
|
|
61
|
-
"async": "^3.2.
|
|
62
|
-
"chai": "^
|
|
63
|
-
"chai-as-promised": "^
|
|
64
|
-
"
|
|
65
|
-
"commander": "^9.4.1",
|
|
59
|
+
"archiver": "^7.0.1",
|
|
60
|
+
"async": "^3.2.6",
|
|
61
|
+
"chai": "^6.0.1",
|
|
62
|
+
"chai-as-promised": "^8.0.2",
|
|
63
|
+
"commander": "^14.0.1",
|
|
66
64
|
"config": "^4.1.1",
|
|
67
|
-
"croner": "^
|
|
68
|
-
"cronstrue": "^
|
|
69
|
-
"cross-env": "^
|
|
70
|
-
"
|
|
71
|
-
"dotenv": "^10.0.0",
|
|
65
|
+
"croner": "^9.1.0",
|
|
66
|
+
"cronstrue": "^3.3.0",
|
|
67
|
+
"cross-env": "^10.0.0",
|
|
68
|
+
"dotenv": "^17.2.2",
|
|
72
69
|
"eslint": "^8.53.0",
|
|
73
70
|
"eslint-config-airbnb-base": "^15.0.0",
|
|
74
|
-
"eslint-plugin-chai-friendly": "^
|
|
75
|
-
"eslint-plugin-import": "^2.
|
|
76
|
-
"eslint-plugin-jsdoc": "^
|
|
71
|
+
"eslint-plugin-chai-friendly": "^1.1.0",
|
|
72
|
+
"eslint-plugin-import": "^2.32.0",
|
|
73
|
+
"eslint-plugin-jsdoc": "^58.1.1",
|
|
77
74
|
"eslint-plugin-json-format": "^2.0.1",
|
|
78
|
-
"eslint-plugin-no-only-tests": "^3.
|
|
79
|
-
"express": "^
|
|
80
|
-
"express-async-errors": "^3.1.1",
|
|
75
|
+
"eslint-plugin-no-only-tests": "^3.3.0",
|
|
76
|
+
"express": "^5.1.0",
|
|
81
77
|
"fetch-charset-detection": "^1.0.1",
|
|
82
|
-
"
|
|
83
|
-
"helmet": "^
|
|
84
|
-
"http-proxy-agent": "^
|
|
85
|
-
"https": "^
|
|
86
|
-
"
|
|
87
|
-
"iconv-lite": "^0.6.3",
|
|
78
|
+
"form-data": "^4.0.4",
|
|
79
|
+
"helmet": "^8.1.0",
|
|
80
|
+
"http-proxy-agent": "^7.0.2",
|
|
81
|
+
"https-proxy-agent": "^7.0.6",
|
|
82
|
+
"iconv-lite": "^0.7.0",
|
|
88
83
|
"joplin-turndown-plugin-gfm": "^1.0.12",
|
|
89
84
|
"js-yaml": "^4.1.0",
|
|
90
|
-
"jsdom": "^
|
|
85
|
+
"jsdom": "^27.0.0",
|
|
91
86
|
"json-source-map": "^0.6.1",
|
|
92
|
-
"
|
|
93
|
-
"
|
|
94
|
-
"
|
|
95
|
-
"
|
|
96
|
-
"
|
|
97
|
-
"
|
|
98
|
-
"
|
|
99
|
-
"pdfjs-dist": "^2.9.359",
|
|
100
|
-
"puppeteer": "^22.8.1",
|
|
87
|
+
"mime": "^4.1.0",
|
|
88
|
+
"mocha": "^11.7.2",
|
|
89
|
+
"mongodb": "^6.19.0",
|
|
90
|
+
"morgan": "^1.10.1",
|
|
91
|
+
"node-fetch": "^3.3.2",
|
|
92
|
+
"octokit": "^5.0.3",
|
|
93
|
+
"puppeteer": "^24.22.0",
|
|
101
94
|
"puppeteer-extra": "^3.3.6",
|
|
102
95
|
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
103
|
-
"sib-api-v3-sdk": "^8.
|
|
104
|
-
"simple-git": "^3.
|
|
96
|
+
"sib-api-v3-sdk": "^8.5.0",
|
|
97
|
+
"simple-git": "^3.28.0",
|
|
105
98
|
"swagger-jsdoc": "^6.2.8",
|
|
106
|
-
"swagger-ui-express": "^5.0.
|
|
107
|
-
"
|
|
99
|
+
"swagger-ui-express": "^5.0.1",
|
|
100
|
+
"turndown": "^7.2.1",
|
|
101
|
+
"winston": "^3.17.0",
|
|
108
102
|
"winston-mail": "^2.0.0"
|
|
109
103
|
},
|
|
110
104
|
"devDependencies": {
|
|
111
|
-
"@commitlint/cli": "^19.
|
|
112
|
-
"dir-compare": "^
|
|
113
|
-
"
|
|
114
|
-
"nock": "^13.2.1",
|
|
105
|
+
"@commitlint/cli": "^19.8.1",
|
|
106
|
+
"dir-compare": "^5.0.0",
|
|
107
|
+
"nock": "^14.0.10",
|
|
115
108
|
"node-stream-zip": "^1.15.0",
|
|
116
|
-
"
|
|
117
|
-
"
|
|
118
|
-
"
|
|
119
|
-
"sinon-chai": "^3.7.0",
|
|
120
|
-
"supertest": "^6.3.3"
|
|
109
|
+
"sinon": "^21.0.0",
|
|
110
|
+
"sinon-chai": "^4.0.1",
|
|
111
|
+
"supertest": "^7.1.4"
|
|
121
112
|
},
|
|
122
113
|
"peerDependencies": {
|
|
123
|
-
"@opentermsarchive/terms-types": "^2.
|
|
114
|
+
"@opentermsarchive/terms-types": "^2.1.0"
|
|
124
115
|
},
|
|
125
116
|
"engines": {
|
|
126
|
-
"node": ">=
|
|
117
|
+
"node": ">=20.0.0 < 23.0.0"
|
|
127
118
|
}
|
|
128
119
|
}
|
|
@@ -2,7 +2,7 @@ import fs from 'fs/promises';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import { expect, use } from 'chai';
|
|
6
6
|
import config from 'config';
|
|
7
7
|
import dircompare from 'dir-compare';
|
|
8
8
|
import mime from 'mime';
|
|
@@ -15,8 +15,6 @@ import generateArchive from './index.js';
|
|
|
15
15
|
|
|
16
16
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
17
17
|
|
|
18
|
-
const { expect } = chai;
|
|
19
|
-
|
|
20
18
|
const FIRST_SERVICE_PROVIDER_ID = 'ServiceA';
|
|
21
19
|
const SECOND_SERVICE_PROVIDER_ID = 'ServiceB';
|
|
22
20
|
|
|
@@ -113,7 +111,7 @@ describe('Export', () => {
|
|
|
113
111
|
});
|
|
114
112
|
});
|
|
115
113
|
|
|
116
|
-
|
|
114
|
+
use(chai => {
|
|
117
115
|
const { Assertion } = chai;
|
|
118
116
|
|
|
119
117
|
Assertion.addMethod('sameContentAs', function (expectedContentPath) {
|
|
@@ -10,7 +10,7 @@ import GitLab from '../../../../src/reporter/gitlab/index.js';
|
|
|
10
10
|
import * as readme from '../../assets/README.template.js';
|
|
11
11
|
import logger from '../../logger/index.js';
|
|
12
12
|
|
|
13
|
-
dotenv.config();
|
|
13
|
+
dotenv.config({ quiet: true });
|
|
14
14
|
|
|
15
15
|
export default async function publish({
|
|
16
16
|
archivePath,
|
|
@@ -4,7 +4,7 @@ import path from 'path';
|
|
|
4
4
|
import Ajv from 'ajv';
|
|
5
5
|
import addFormats from 'ajv-formats';
|
|
6
6
|
import config from 'config';
|
|
7
|
-
import
|
|
7
|
+
import { Cron } from 'croner';
|
|
8
8
|
import yaml from 'js-yaml';
|
|
9
9
|
|
|
10
10
|
import specsRouter from '../../src/collection-api/routes/docs.js';
|
|
@@ -15,7 +15,7 @@ describe('Metadata file validation', () => {
|
|
|
15
15
|
'iso3166-2': code => /^[A-Z]{2}(-[A-Z0-9]{1,3})?$/.test(code),
|
|
16
16
|
'cron-expression': cronExpression => {
|
|
17
17
|
try {
|
|
18
|
-
|
|
18
|
+
new Cron(cronExpression); // eslint-disable-line no-new
|
|
19
19
|
|
|
20
20
|
return true;
|
|
21
21
|
} catch {
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import 'dotenv/config';
|
|
2
1
|
import config from 'config';
|
|
3
|
-
import
|
|
2
|
+
import dotenv from 'dotenv';
|
|
3
|
+
import { Octokit } from 'octokit'; // eslint-disable-line import/no-unresolved
|
|
4
|
+
|
|
5
|
+
dotenv.config({ quiet: true });
|
|
4
6
|
|
|
5
7
|
async function removeDuplicateIssues() {
|
|
6
8
|
const repository = config.get('@opentermsarchive/engine.reporter.githubIssues.repositories.declarations');
|
|
@@ -2,7 +2,7 @@ import fsApi from 'fs';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import { expect } from 'chai';
|
|
6
6
|
import mime from 'mime';
|
|
7
7
|
|
|
8
8
|
import SourceDocument from '../services/sourceDocument.js';
|
|
@@ -13,7 +13,6 @@ import extract from './index.js';
|
|
|
13
13
|
|
|
14
14
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
15
15
|
const fs = fsApi.promises;
|
|
16
|
-
const { expect } = chai;
|
|
17
16
|
|
|
18
17
|
const virtualLocation = 'https://example.com/main';
|
|
19
18
|
const rawHTML = `
|
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import ciceroMark from '@accordproject/markdown-cicero';
|
|
2
2
|
import mardownPdf from '@accordproject/markdown-pdf';
|
|
3
|
-
import TurndownService from '@opentermsarchive/turndown';
|
|
4
3
|
import turndownPluginGithubFlavouredMarkdown from 'joplin-turndown-plugin-gfm';
|
|
4
|
+
import TurndownService from 'turndown';
|
|
5
5
|
|
|
6
6
|
const turndownService = new TurndownService();
|
|
7
7
|
|
|
8
8
|
turndownService.use(turndownPluginGithubFlavouredMarkdown.gfm);
|
|
9
9
|
|
|
10
|
+
turndownService.addRule('fix-anchors-containing-block-elements', {
|
|
11
|
+
...turndownService.options.rules.inlineLink,
|
|
12
|
+
replacement: (content, node) => turndownService.options.rules.inlineLink.replacement(content.trim(), node),
|
|
13
|
+
});
|
|
14
|
+
|
|
10
15
|
const { PdfTransformer } = mardownPdf;
|
|
11
16
|
const { CiceroMarkTransformer } = ciceroMark;
|
|
12
17
|
const ciceroMarkTransformer = new CiceroMarkTransformer();
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import http from 'http';
|
|
2
2
|
|
|
3
|
-
import
|
|
3
|
+
import { expect, use } from 'chai';
|
|
4
4
|
import chaiAsPromised from 'chai-as-promised';
|
|
5
5
|
|
|
6
6
|
import fetch, { launchHeadlessBrowser, stopHeadlessBrowser } from './fullDomFetcher.js';
|
|
7
7
|
|
|
8
|
-
const { expect } = chai;
|
|
9
8
|
const SERVER_PORT = 8977;
|
|
10
9
|
|
|
11
|
-
|
|
10
|
+
use(chaiAsPromised);
|
|
12
11
|
|
|
13
12
|
const dynamicHTML = '<!DOCTYPE html><html><head><title>Dynamic Page</title><script>setTimeout(() => { document.body.innerHTML += "<div class=\'dynamic\'>Loaded</div>"; }, 100);</script></head><body></body></html>';
|
|
14
13
|
const delayedContentHTML = '<!DOCTYPE html><html><head><title>Delayed Content</title><script>setTimeout(() => { document.querySelector(".content").textContent = "Final content"; }, 100);</script></head><body><div class="content"></div></body></html>';
|
|
@@ -3,7 +3,7 @@ import http from 'http';
|
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import { fileURLToPath } from 'url';
|
|
5
5
|
|
|
6
|
-
import
|
|
6
|
+
import { expect, use } from 'chai';
|
|
7
7
|
import chaiAsPromised from 'chai-as-promised';
|
|
8
8
|
import iconv from 'iconv-lite';
|
|
9
9
|
|
|
@@ -11,10 +11,9 @@ import fetch, { launchHeadlessBrowser, stopHeadlessBrowser, FetchDocumentError,
|
|
|
11
11
|
|
|
12
12
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
13
13
|
|
|
14
|
-
const { expect } = chai;
|
|
15
14
|
const SERVER_PORT = 8976;
|
|
16
15
|
|
|
17
|
-
|
|
16
|
+
use(chaiAsPromised);
|
|
18
17
|
|
|
19
18
|
const termsHTML = '<!DOCTYPE html><html><head><meta charset="UTF-8"><title>First provider TOS</title></head><body><h1>Terms of service</h1><p>Dapibus quis diam sagittis</p></body></html>';
|
|
20
19
|
const termsWithOtherCharsetHTML = '<!DOCTYPE html><html><head><meta http-equiv="Content-Type" content="text/html; charset=windows-1251"><title>TOS на първия доставчик</title></head><body><h1>Условия за ползване</h1><p>Dapibus quis diam sagittis</p></body></html>';
|
|
@@ -2,7 +2,7 @@ import fsApi from 'fs';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import { expect, use } from 'chai';
|
|
6
6
|
import config from 'config';
|
|
7
7
|
import nock from 'nock';
|
|
8
8
|
import sinon from 'sinon';
|
|
@@ -16,8 +16,7 @@ import Archivist, { EVENTS } from './index.js';
|
|
|
16
16
|
|
|
17
17
|
const fs = fsApi.promises;
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
const { expect } = chai;
|
|
19
|
+
use(sinonChai);
|
|
21
20
|
|
|
22
21
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
23
22
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { expect } from 'chai';
|
|
2
2
|
import config from 'config';
|
|
3
3
|
|
|
4
4
|
import Snapshot from './snapshot.js';
|
|
@@ -6,8 +6,6 @@ import Version from './version.js';
|
|
|
6
6
|
|
|
7
7
|
import Recorder from './index.js';
|
|
8
8
|
|
|
9
|
-
const { expect } = chai;
|
|
10
|
-
|
|
11
9
|
const MIME_TYPE = 'text/html';
|
|
12
10
|
const FETCH_DATE = new Date('2000-01-01T12:00:00.000Z');
|
|
13
11
|
const FETCH_DATE_LATER = new Date('2000-01-02T12:00:00.000Z');
|
|
@@ -5,8 +5,6 @@ import mime from 'mime';
|
|
|
5
5
|
import Snapshot from '../../snapshot.js';
|
|
6
6
|
import Version from '../../version.js';
|
|
7
7
|
|
|
8
|
-
mime.define({ 'text/markdown': ['md'] }, true); // ensure extension for markdown files is `.md` and not `.markdown`
|
|
9
|
-
|
|
10
8
|
export const COMMIT_MESSAGE_PREFIXES = {
|
|
11
9
|
startTracking: 'First record of',
|
|
12
10
|
extractOnly: 'Apply technical or declaration upgrade on',
|
|
@@ -15,8 +15,6 @@ import Git from './git.js';
|
|
|
15
15
|
|
|
16
16
|
const fs = fsApi.promises;
|
|
17
17
|
|
|
18
|
-
mime.define({ 'text/markdown': ['md'] }, true); // ensure extension for markdown files is `.md` and not `.markdown`
|
|
19
|
-
|
|
20
18
|
export default class GitRepository extends RepositoryInterface {
|
|
21
19
|
constructor({ path, author, publish, snapshotIdentiferTemplate }) {
|
|
22
20
|
super();
|
|
@@ -2,7 +2,7 @@ import fs from 'fs';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import { expect } from 'chai';
|
|
6
6
|
import config from 'config';
|
|
7
7
|
import mime from 'mime';
|
|
8
8
|
|
|
@@ -14,8 +14,6 @@ import Git from './git.js';
|
|
|
14
14
|
|
|
15
15
|
import GitRepository from './index.js';
|
|
16
16
|
|
|
17
|
-
const { expect } = chai;
|
|
18
|
-
|
|
19
17
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
20
18
|
|
|
21
19
|
const RECORDER_PATH = path.resolve(__dirname, '../../../../..', config.get('@opentermsarchive/engine.recorder.versions.storage.git.path'));
|
|
@@ -79,7 +79,11 @@ export default class MongoRepository extends RepositoryInterface {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
async findById(recordId) {
|
|
82
|
-
|
|
82
|
+
if (!ObjectId.isValid(recordId)) {
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const mongoDocument = await this.collection.findOne({ _id: ObjectId.createFromHexString(recordId) });
|
|
83
87
|
|
|
84
88
|
return this.#toDomain(mongoDocument);
|
|
85
89
|
}
|
|
@@ -108,7 +112,7 @@ export default class MongoRepository extends RepositoryInterface {
|
|
|
108
112
|
}
|
|
109
113
|
|
|
110
114
|
async loadRecordContent(record) {
|
|
111
|
-
const { content } = await this.collection.findOne({ _id:
|
|
115
|
+
const { content } = await this.collection.findOne({ _id: ObjectId.createFromHexString(record.id) }, { projection: { content: 1 } });
|
|
112
116
|
|
|
113
117
|
record.content = content instanceof Binary ? content.buffer : content;
|
|
114
118
|
}
|
|
@@ -2,17 +2,16 @@ import fs from 'fs';
|
|
|
2
2
|
import path from 'path';
|
|
3
3
|
import { fileURLToPath } from 'url';
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import { expect } from 'chai';
|
|
6
6
|
import config from 'config';
|
|
7
7
|
import mime from 'mime';
|
|
8
|
-
import { MongoClient } from 'mongodb';
|
|
8
|
+
import { MongoClient, ObjectId } from 'mongodb';
|
|
9
9
|
|
|
10
10
|
import Snapshot from '../../snapshot.js';
|
|
11
11
|
import Version from '../../version.js';
|
|
12
12
|
|
|
13
13
|
import MongoRepository from './index.js';
|
|
14
14
|
|
|
15
|
-
const { expect } = chai;
|
|
16
15
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
17
16
|
|
|
18
17
|
const { connectionURI } = config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo');
|
|
@@ -54,6 +53,10 @@ describe('MongoRepository', () => {
|
|
|
54
53
|
collection = db.collection(config.get('@opentermsarchive/engine.recorder.versions.storage.mongo.collection'));
|
|
55
54
|
});
|
|
56
55
|
|
|
56
|
+
after(async () => {
|
|
57
|
+
await client.close();
|
|
58
|
+
});
|
|
59
|
+
|
|
57
60
|
describe('#save', () => {
|
|
58
61
|
let record;
|
|
59
62
|
let mongoDocument;
|
|
@@ -444,7 +447,9 @@ describe('MongoRepository', () => {
|
|
|
444
447
|
|
|
445
448
|
context('when requested record does not exist', () => {
|
|
446
449
|
it('returns null', async () => {
|
|
447
|
-
|
|
450
|
+
const nonExistentId = new ObjectId().toString();
|
|
451
|
+
|
|
452
|
+
expect(await subject.findById(nonExistentId)).to.equal(null);
|
|
448
453
|
});
|
|
449
454
|
});
|
|
450
455
|
});
|
|
@@ -840,6 +845,10 @@ describe('MongoRepository', () => {
|
|
|
840
845
|
collection = db.collection(config.get('@opentermsarchive/engine.recorder.snapshots.storage.mongo.collection'));
|
|
841
846
|
});
|
|
842
847
|
|
|
848
|
+
after(async () => {
|
|
849
|
+
await client.close();
|
|
850
|
+
});
|
|
851
|
+
|
|
843
852
|
describe('#save', () => {
|
|
844
853
|
let record;
|
|
845
854
|
let mongoDocument;
|
|
@@ -1119,7 +1128,9 @@ describe('MongoRepository', () => {
|
|
|
1119
1128
|
|
|
1120
1129
|
context('when requested record does not exist', () => {
|
|
1121
1130
|
it('returns null', async () => {
|
|
1122
|
-
|
|
1131
|
+
const nonExistentId = new ObjectId().toString();
|
|
1132
|
+
|
|
1133
|
+
expect(await subject.findById(nonExistentId)).to.equal(null);
|
|
1123
1134
|
});
|
|
1124
1135
|
});
|
|
1125
1136
|
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'fs/promises';
|
|
2
2
|
import path from 'path';
|
|
3
3
|
|
|
4
|
-
import
|
|
4
|
+
import { expect, use } from 'chai';
|
|
5
5
|
import sinon from 'sinon';
|
|
6
6
|
import sinonChai from 'sinon-chai';
|
|
7
7
|
|
|
@@ -14,8 +14,7 @@ import Terms from './terms.js';
|
|
|
14
14
|
|
|
15
15
|
import { getDeclaredServicesIds, loadServiceDeclaration, loadServiceFilters, getServiceFilters, createSourceDocuments, createServiceFromDeclaration, load, loadWithHistory } from './index.js';
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
const { expect } = chai;
|
|
17
|
+
use(sinonChai);
|
|
19
18
|
|
|
20
19
|
describe('Services', () => {
|
|
21
20
|
describe('#getDeclaredServicesIds', () => {
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { expect } from 'chai';
|
|
2
2
|
|
|
3
3
|
import Service from './service.js';
|
|
4
4
|
import Terms from './terms.js';
|
|
5
5
|
|
|
6
|
-
const { expect } = chai;
|
|
7
|
-
|
|
8
6
|
describe('Service', () => {
|
|
9
7
|
let subject;
|
|
10
8
|
const TERMS_OF_SERVICE_TYPE = 'Terms of Service';
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { expect } from 'chai';
|
|
2
2
|
|
|
3
3
|
import SourceDocument from './sourceDocument.js';
|
|
4
4
|
import Terms from './terms.js';
|
|
5
5
|
|
|
6
|
-
const { expect } = chai;
|
|
7
|
-
|
|
8
6
|
describe('Terms', () => {
|
|
9
7
|
const service = { name: 'Service' };
|
|
10
8
|
const termsType = 'Terms of Service';
|
|
@@ -155,10 +155,10 @@ const { version: PACKAGE_VERSION } = require('../../../package.json');
|
|
|
155
155
|
* roles:
|
|
156
156
|
* type: array
|
|
157
157
|
* description: Roles of the entity within the governance, see [collection governance](https://docs.opentermsarchive.org/collections/reference/governance/)
|
|
158
|
-
* example: [host,
|
|
158
|
+
* example: [host, sysadmin]
|
|
159
159
|
* items:
|
|
160
160
|
* type: string
|
|
161
|
-
* enum: [host,
|
|
161
|
+
* enum: [host, sysadmin, curator, maintainer, contributor, analyst, diffuser, sponsor]
|
|
162
162
|
* i18n:
|
|
163
163
|
* type: object
|
|
164
164
|
* description: Internationalization of any of the Metadata properties (except i18n itself) for different language codes
|
|
@@ -30,38 +30,28 @@ const EXPECTED_RESPONSE = {
|
|
|
30
30
|
jurisdictions: [
|
|
31
31
|
'EU',
|
|
32
32
|
],
|
|
33
|
+
trackingPeriods: [
|
|
34
|
+
{
|
|
35
|
+
startDate: '2025-07-17',
|
|
36
|
+
schedule: '30 */12 * * *',
|
|
37
|
+
serverLocation: 'Strasbourg, FR',
|
|
38
|
+
},
|
|
39
|
+
],
|
|
33
40
|
governance: {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
|
|
49
|
-
},
|
|
50
|
-
],
|
|
51
|
-
maintainers: [
|
|
52
|
-
{
|
|
53
|
-
name: 'Open Terms Archive',
|
|
54
|
-
url: 'https://opentermsarchive.org/',
|
|
55
|
-
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
|
|
56
|
-
},
|
|
57
|
-
],
|
|
58
|
-
sponsors: [
|
|
59
|
-
{
|
|
60
|
-
name: 'Open Terms Archive',
|
|
61
|
-
url: 'https://opentermsarchive.org/',
|
|
62
|
-
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
|
|
63
|
-
},
|
|
64
|
-
],
|
|
41
|
+
'Open Terms Archive': {
|
|
42
|
+
url: 'https://opentermsarchive.org',
|
|
43
|
+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
|
|
44
|
+
roles: [
|
|
45
|
+
'host',
|
|
46
|
+
'sysadmin',
|
|
47
|
+
'curator',
|
|
48
|
+
'maintainer',
|
|
49
|
+
'contributor',
|
|
50
|
+
'analyst',
|
|
51
|
+
'diffuser',
|
|
52
|
+
'sponsor',
|
|
53
|
+
],
|
|
54
|
+
},
|
|
65
55
|
},
|
|
66
56
|
};
|
|
67
57
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { expect } from 'chai';
|
|
2
2
|
import config from 'config';
|
|
3
3
|
import supertest from 'supertest';
|
|
4
4
|
|
|
@@ -9,7 +9,6 @@ import app from '../server.js';
|
|
|
9
9
|
|
|
10
10
|
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
|
|
11
11
|
|
|
12
|
-
const { expect } = chai;
|
|
13
12
|
const request = supertest(app);
|
|
14
13
|
|
|
15
14
|
describe('Versions API', () => {
|
package/src/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createRequire } from 'module';
|
|
2
2
|
|
|
3
3
|
import config from 'config';
|
|
4
|
-
import
|
|
4
|
+
import { Cron } from 'croner';
|
|
5
5
|
import cronstrue from 'cronstrue';
|
|
6
6
|
|
|
7
7
|
import { getCollection } from './archivist/collection/index.js';
|
|
@@ -83,7 +83,7 @@ export default async function track({ services, types, extractOnly, schedule })
|
|
|
83
83
|
logger.info('The scheduler is running…');
|
|
84
84
|
logger.info(`Terms will be tracked ${humanReadableSchedule.toLowerCase()} in the timezone of this machine`);
|
|
85
85
|
|
|
86
|
-
|
|
86
|
+
new Cron( // eslint-disable-line no-new
|
|
87
87
|
trackingSchedule,
|
|
88
88
|
{ protect: job => logger.warn(`Tracking scheduled at ${new Date().toISOString()} were blocked by an unfinished tracking started at ${job.currentRun().toISOString()}`) },
|
|
89
89
|
() => archivist.track({ services, types }),
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire } from 'module';
|
|
2
2
|
|
|
3
|
-
import { Octokit } from 'octokit';
|
|
3
|
+
import { Octokit } from 'octokit'; // eslint-disable-line import/no-unresolved
|
|
4
4
|
|
|
5
5
|
import logger from '../../logger/index.js';
|
|
6
6
|
import { LABELS, MANAGED_BY_OTA_MARKER, DEPRECATED_MANAGED_BY_OTA_MARKER } from '../labels.js';
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { expect } from 'chai';
|
|
2
2
|
|
|
3
3
|
import GitHub from './github/index.js';
|
|
4
4
|
import GitLab from './gitlab/index.js';
|
|
5
5
|
import { MANAGED_BY_OTA_MARKER, LABELS } from './labels.js';
|
|
6
6
|
|
|
7
|
-
const { expect } = chai;
|
|
8
|
-
|
|
9
7
|
const MAX_LABEL_DESCRIPTION_LENGTH = Math.min(
|
|
10
8
|
GitHub.MAX_LABEL_DESCRIPTION_LENGTH,
|
|
11
9
|
GitLab.MAX_LABEL_DESCRIPTION_LENGTH,
|