@opentermsarchive/engine 0.15.0

package/ops/roles/ota/files/.env

@@ -0,0 +1,21 @@
+$ANSIBLE_VAULT;1.1;AES256
+65653333633465346264373663643062363265383439323061633338323166393262663332646534
+3034646331313536623831616466653564396663636166650a643831383438613765323264386266
+37383663326333636366653537383339636231313261323239663164393933366535623730313337
+3230323166306336630a646239643135326335393364366534663535316334333531353431323331
+62376464333830643539333764646534613935386263633634623962653537363139363265353864
+34623031333535313930633964626438343432363366633263396466323830323035343333373763
+31303962353238383666343834653539663536623561393966373763393535663038333833396433
+61326137363133373861343732336335373037613134336362396563653563646364376438366363
+61613437393364396332313833356138303835633739653937656631313737623762346235316164
+30353961393663663635393630653535353064353031623962333038366638376333646130663263
+35346637303534663639613537316538663134356166323566613333343236383631653563643433
+65373533656236613631653665383039303564613830343530363130663462343665343865666637
+61353766623634303537326163623564663433343835366463613837303361393538656365633365
+64306532626533373561633066616336663439653734313832366333323461383065326166313635
+38343261656638396235326266376531633662343235633336363361313266643166346433356539
+38363833336562623631306432656531326465306535623164353035366339336663626161613364
+39616539616532643935623939313834343964303035383334666261363336303034346439316663
+65306530333965353766363336386564383932333234303334633438333839316362666533363330
+34336232623333353366363232656461646236383061383634653137623661376639323162383162
+38653230353662316366

package/ops/roles/ota/tasks/database.yml

@@ -0,0 +1,65 @@
+- name: Check if {{ name }} base data has already been obtained from {{ repository }}
+  git:
+    repo: '{{ repository }}'
+    version: '{{ branch }}'
+    dest: '/home/{{ ansible_user }}/{{ directory }}'
+    clone: no
+    update: no
+    accept_hostkey: yes
+    key_file: '/home/{{ ansible_user }}/.ssh/ota-bot-key'
+  register: existing_repository # the `before` property of the return value can tell us if the repository has been cloned already or not, see <https://docs.ansible.com/ansible/latest/collections/ansible/builtin/git_module.html#return-values>
+  tags:
+    - restart
+    - start
+    - update
+    - setup
+
+- name: Obtain {{ name }} initial data from branch {{ branch }} of {{ repository }}
+  git:
+    repo: '{{ repository }}'
+    version: '{{ branch }}'
+    dest: '/home/{{ ansible_user }}/{{ directory }}'
+    accept_hostkey: yes
+    key_file: '/home/{{ ansible_user }}/.ssh/ota-bot-key'
+  when: existing_repository.before is defined and not existing_repository.before # if existing_repository.before is null, then the repository is new
+  tags:
+    - setup
+
+- name: Remove existing locks in {{ name }}
+  file:
+    path: '/home/{{ ansible_user }}/{{ directory }}/.git/index.lock'
+    state: absent
+  tags:
+    - restart
+    - start
+    - update
+
+- name: Get latest data from {{ repository }}
+  command:
+    cmd: git fetch origin
+    chdir: '/home/{{ ansible_user }}/{{ directory }}'
+  tags:
+    - restart
+    - start
+    - update
+    - setup
+
+- name: Clean {{ name }} local copy
+  command:
+    cmd: git reset --hard origin/{{ branch }}
+    chdir: '/home/{{ ansible_user }}/{{ directory }}'
+  tags:
+    - restart
+    - start
+    - update
+    - setup
+
+- name: Ensure {{ name }} is on branch {{ branch }}
+  command:
+    cmd: git checkout {{ branch }}
+    chdir: '/home/{{ ansible_user }}/{{ directory }}'
+  tags:
+    - restart
+    - start
+    - update
+    - setup

package/ops/roles/ota/tasks/main.yml

@@ -0,0 +1,110 @@
+- name: Clone Open Terms Archive engine
+  git:
+    repo: '{{ ota_repository }}'
+    dest: '/home/{{ ansible_user }}/{{ ota_directory }}'
+    version: '{{ ota_branch }}'
+    force: yes
+    accept_hostkey: yes
+    key_file: '/home/{{ ansible_user }}/.ssh/ota-bot-key'
+    depth: 1
+  tags:
+    - setup
+    - update
+
+- name: Install dependencies
+  command:
+    cmd: npm ci --production
+    chdir: '/home/{{ ansible_user }}/{{ ota_directory }}'
+  tags:
+    - setup
+    - update
+
+- name: Add .env file
+  copy:
+    src: .env
+    dest: '/home/{{ ansible_user }}/{{ ota_directory }}/.env'
+  tags:
+    - setup
+
+- name: Load app config
+  ansible.builtin.include_vars:
+    name: app_config
+    file: '../config/{{ config_file_name }}.json'
+  tags:
+    - setup
+    - update
+    - update-declarations
+
+- name: Make config available to app
+  ansible.builtin.file:
+    src: '/home/{{ ansible_user }}/{{ ota_directory }}/config/{{ config_file_name }}.json'
+    dest: '/home/{{ ansible_user }}/{{ ota_directory }}/config/local.json'
+    state: link
+
+- name: Install services declarations
+  git:
+    repo: '{{ app_config.services.repository }}'
+    dest: '/home/{{ ansible_user }}/{{ declarations_directory }}'
+    version: '{{ declarations_branch }}'
+    force: yes
+    accept_hostkey: yes
+    key_file: '/home/{{ ansible_user }}/.ssh/ota-bot-key'
+    depth: 1
+  tags:
+    - setup
+    - update
+    - update-declarations
+
+- name: Install services declarations dependencies
+  command:
+    cmd: npm install --production
+    chdir: '/home/{{ ansible_user }}/{{ declarations_directory }}'
+  tags:
+    - setup
+    - update
+    - update-declarations
+
+- name: Stop Open Terms Archive schedulers
+  command:
+    cmd: pm2 stop pm2.config.cjs
+    chdir: '/home/{{ ansible_user }}/{{ ota_directory }}'
+  tags:
+    - stop
+    - update
+    - update-declarations
+
+- include_tasks: database.yml
+  name: Setup snapshots git repository
+  when: app_config.recorder.snapshots.storage.git.repository is defined
+  vars:
+    name: snapshots
+    repository: '{{ app_config.recorder.snapshots.storage.git.repository }}'
+    branch: '{{ snapshots_branch }}'
+    directory: '{{ snapshots_directory }}'
+  tags:
+    - setup
+    - update
+
+- include_tasks: database.yml
+  name: Setup versions git repository
+  when: app_config.recorder.versions.storage.git.repository is defined
+  vars:
+    name: versions
+    repository: '{{ app_config.recorder.versions.storage.git.repository }}'
+    branch: '{{ versions_branch }}'
+    directory: '{{ versions_directory }}'
+  tags:
+    - setup
+    - update
+
+- name: Start Open Terms Archive schedulers
+  command:
+    cmd: pm2 startOrRestart pm2.config.cjs
+    chdir: '/home/{{ ansible_user }}/{{ ota_directory }}'
+  environment:
+    NODE_ENV: production
+  tags:
+    - restart
+    - start
+    - update
+    - update-declarations

package/ops/site.yml

@@ -0,0 +1,6 @@
+---
+- name: Set up Open Terms Archive
+  hosts: all
+
+- import_playbook: infra.yml
+- import_playbook: app.yml

package/package.json

@@ -0,0 +1,101 @@
+{
+  "name": "@opentermsarchive/engine",
+  "version": "0.15.0",
+  "description": "Tracks and makes visible changes to the terms of online services",
+  "homepage": "https://github.com/ambanum/OpenTermsArchive#readme",
+  "bugs": {
+    "url": "https://github.com/ambanum/OpenTermsArchive/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ambanum/OpenTermsArchive.git"
+  },
+  "license": "EUPL-1.2",
+  "author": "ambanum",
+  "type": "module",
+  "exports": {
+    ".": "./src/exports.js",
+    "./fetch": "./src/archivist/fetcher/exports.js",
+    "./filter": "./src/archivist/filter/exports.js",
+    "./page-declaration": "./src/archivist/services/pageDeclaration.js"
+  },
+  "bin": {
+    "ota-lint-declarations": "./bin/lint-declarations.js",
+    "ota-track": "./bin/track.js",
+    "ota-validate-declarations": "./bin/validate-declarations.js"
+  },
+  "scripts": {
+    "dataset:generate": "node scripts/dataset/main.js",
+    "dataset:release": "node scripts/dataset/main.js --publish --remove-local-copy",
+    "dataset:scheduler": "npm run dataset:release -- --schedule",
+    "declarations:lint": "node ./bin/lint-declarations.js",
+    "declarations:validate": "node ./bin/validate-declarations.js",
+    "declarations:validate:schema": "npm run declarations:validate -- --schema-only",
+    "lint": "eslint src test scripts bin",
+    "lint:fix": "npm run lint -- --fix",
+    "refilter": "npm start -- --refilter-only",
+    "start": "node --max-http-header-size=32768 src/main.js",
+    "start:scheduler": "npm start -- --schedule",
+    "test": "cross-env NODE_ENV=test mocha --recursive \"./src/**/*.test.js\" \"./scripts/**/*.test.js\" --exit",
+    "posttest": "npm run lint",
+    "test:debug": "npm run test -- --inspect-brk --exit"
+  },
+  "dependencies": {
+    "@accordproject/markdown-cicero": "^0.15.2",
+    "@accordproject/markdown-pdf": "^0.15.2",
+    "@opentermsarchive/fetch-charset-detection": "^1.0.1",
+    "@opentermsarchive/turndown": "^7.1.3",
+    "abort-controller": "^3.0.0",
+    "ajv": "^6.12.6",
+    "archiver": "^5.3.0",
+    "async": "^3.2.2",
+    "chai": "^4.3.4",
+    "chai-as-promised": "^7.1.1",
+    "chai-exclude": "^2.1.0",
+    "commander": "^9.4.1",
+    "config": "^3.3.6",
+    "croner": "^4.3.6",
+    "cross-env": "^7.0.3",
+    "datauri": "^4.1.0",
+    "deep-diff": "^1.0.2",
+    "dotenv": "^10.0.0",
+    "eslint": "^8.2.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-plugin-chai-friendly": "^0.7.2",
+    "eslint-plugin-import": "^2.25.3",
+    "eslint-plugin-json-format": "^2.0.1",
+    "fs-extra": "^10.0.0",
+    "http-proxy-agent": "^5.0.0",
+    "https": "^1.0.0",
+    "https-proxy-agent": "^5.0.0",
+    "iconv-lite": "^0.6.3",
+    "joplin-turndown-plugin-gfm": "^1.0.12",
+    "jsdom": "^18.1.0",
+    "json-source-map": "^0.6.1",
+    "lodash": "^4.17.21",
+    "mime": "^2.5.2",
+    "mocha": "^9.1.3",
+    "mongodb": "^4.9.0",
+    "node-fetch": "^3.1.0",
+    "octokit": "^1.7.0",
+    "pdfjs-dist": "^2.9.359",
+    "puppeteer": "14.3.0",
+    "puppeteer-extra": "^3.2.3",
+    "puppeteer-extra-plugin-stealth": "^2.9.0",
+    "sib-api-v3-sdk": "^8.2.1",
+    "simple-git": "^3.8.0",
+    "winston": "^3.3.3",
+    "winston-mail": "^2.0.0"
+  },
+  "devDependencies": {
+    "dir-compare": "^4.0.0",
+    "nock": "^13.2.1",
+    "node-stream-zip": "^1.15.0",
+    "prettier": "^2.2.1",
+    "sinon": "^12.0.1",
+    "sinon-chai": "^3.7.0"
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  }
+}

package/pm2.config.cjs

@@ -0,0 +1,20 @@
+module.exports = {
+  apps: [
+    {
+      name: 'ota',
+      script: 'npm',
+      args: 'run start:scheduler',
+      max_restarts: 2,
+      min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable.
+      restart_delay: 15 * 60 * 1000,
+    },
+    {
+      name: 'ota-release',
+      script: 'npm',
+      args: 'run dataset:scheduler',
+      max_restarts: 2,
+      min_uptime: '1h', // Set a relatively high duration (more than the longest run) so that restarts that occur before this duration has elapsed are considered unstable.
+      restart_delay: 15 * 60 * 1000,
+    },
+  ],
+};

package/scripts/dataset/README.md

@@ -0,0 +1,37 @@
+# Dataset release
+
+Export the versions dataset into a ZIP file and publish it to GitHub releases.
+
+## Configuring
+
+You can change the configuration in the appropriate config file in the `config` folder. See the [main README](https://github.com/ambanum/OpenTermsArchive#configuring) for documentation on using the configuration file.
+
+## Running
+
+To export the dataset into a local ZIP file:
+
+```sh
+node scripts/dataset/main.js [$filename]
+```
+
+To export the dataset into a ZIP file and publish it on GitHub releases:
+
+```sh
+node scripts/dataset/main.js --publish
+```
+
+To export, publish the dataset and remove the local copy that was created after it has been uploaded:
+
+```sh
+node scripts/dataset/main.js --publish --remove-local-copy
+```
+
+To schedule export, publishing and local copy removal:
+
+```sh
+node scripts/dataset/main.js --schedule --publish --remove-local-copy
+```
+
+## Adding renaming rules
+
+See the [renamer module documentation](../renamer/README.md).