@opentermsarchive/engine 0.15.0

package/config/dating.json

@@ -0,0 +1,37 @@
+{
+  "services": {
+    "repository": "https://github.com/OpenTermsArchive/dating-declarations.git"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/dating-snapshots/commit/%SNAPSHOT_ID",
+          "repository": "git@github.com:OpenTermsArchive/dating-versions.git"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/dating-snapshots.git"
+        }
+      }
+    }
+  },
+  "notifier": {
+    "sendInBlue": {
+      "updatesListId": 594,
+      "updateTemplateId": 40
+    }
+  },
+  "tracker": {
+    "githubIssues": {
+      "repository": "OpenTermsArchive/dating-declarations"
+    }
+  },
+  "dataset": {
+    "title": "dating",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/dating-versions"
+  }
+}

package/config/default.json

@@ -0,0 +1,71 @@
+{
+  "services": {
+    "declarationsPath": "../declarations/declarations"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "type": "git",
+        "git": {
+          "path": "./data/versions",
+          "publish": false,
+          "snapshotIdentiferTemplate": "./data/snapshots/%SNAPSHOT_ID",
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "type": "git",
+        "git": {
+          "path": "./data/snapshots",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        },
+        "mongo": {
+          "connectionURI": "mongodb://127.0.0.1:27017",
+          "database": "open-terms-archive",
+          "collection": "snapshots"
+        }
+      }
+    }
+  },
+  "fetcher": {
+    "waitForElementsTimeout": 10000,
+    "navigationTimeout": 30000,
+    "language": "en"
+  },
+  "logger": {
+    "smtp": {
+      "host": "smtp-relay.sendinblue.com",
+      "username": "admin@opentermsarchive.org"
+    },
+    "sendMailOnError": false
+  },
+  "notifier": {
+    "sendInBlue": {
+      "updatesListId": 850,
+      "updateTemplateId": 7
+    }
+  },
+  "tracker": {
+    "githubIssues": {
+      "repository": "OpenTermsArchive/sandbox",
+      "label": {
+        "name": "bot-report",
+        "color": "FEF2C0",
+        "description": "Automatically created by Open Terms Archive when a document cannot be fetched"
+      }
+    }
+  },
+  "dataset": {
+    "title": "sandbox",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
+  }
+}

package/config/france.json

@@ -0,0 +1,40 @@
+{
+  "services": {
+    "repository": "https://github.com/OpenTermsArchive/france-declarations.git"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/france-snapshots/commit/%SNAPSHOT_ID",
+          "repository": "git@github.com:OpenTermsArchive/france-versions.git"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/france-snapshots.git"
+        }
+      }
+    }
+  },
+  "fetcher": {
+    "language": "fr"
+  },
+  "notifier": {
+    "sendInBlue": {
+      "updatesListId": 595,
+      "updateTemplateId": 42
+    }
+  },
+  "tracker": {
+    "githubIssues": {
+      "repository": "OpenTermsArchive/france-declarations"
+    }
+  },
+  "dataset": {
+    "title": "france",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/france-versions"
+  }
+}

package/config/p2b-compliance.json

@@ -0,0 +1,40 @@
+{
+  "services": {
+    "repository": "https://github.com/OpenTermsArchive/p2b-compliance-declarations.git"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/p2b-compliance-snapshots/commit/%SNAPSHOT_ID",
+          "repository": "git@github.com:OpenTermsArchive/p2b-compliance-versions.git"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/p2b-compliance-snapshots.git"
+        }
+      }
+    }
+  },
+  "fetcher": {
+    "language": "en"
+  },
+  "notifier": {
+    "sendInBlue": {
+      "updatesListId": 861,
+      "updateTemplateId": 56
+    }
+  },
+  "tracker": {
+    "githubIssues": {
+      "repository": "OpenTermsArchive/p2b-compliance-declarations"
+    }
+  },
+  "dataset": {
+    "title": "p2b-compliance",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/p2b-compliance-versions"
+  }
+}

package/config/pga.json

@@ -0,0 +1,40 @@
+{
+  "services": {
+    "repository": "https://github.com/OpenTermsArchive/pga-declarations.git"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "snapshotIdentiferTemplate": "https://github.com/OpenTermsArchive/pga-snapshots/commit/%SNAPSHOT_ID",
+          "repository": "git@github.com:OpenTermsArchive/pga-versions.git"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/pga-snapshots.git"
+        }
+      }
+    }
+  },
+  "fetcher": {
+    "language": "en"
+  },
+  "notifier": {
+    "sendInBlue": {
+      "updatesListId": 855,
+      "updateTemplateId": 41
+    }
+  },
+  "tracker": {
+    "githubIssues": {
+      "repository": "OpenTermsArchive/pga-declarations"
+    }
+  },
+  "dataset": {
+    "title": "pga",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/pga-versions"
+  }
+}

package/config/production.json

@@ -0,0 +1,27 @@
+{
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "path": "../versions",
+          "publish": true
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "path": "../snapshots",
+          "publish": true
+        }
+      }
+    }
+  },
+  "logger": {
+    "sendMailOnError": {
+      "to": "admin@opentermsarchive.org",
+      "from": "noreply@opentermsarchive.org",
+      "sendWarnings": false
+    }
+  }
+}

package/config/test.json

@@ -0,0 +1,49 @@
+{
+  "services": {
+    "declarationsPath": "./test/services"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "path": "./test/data/versions",
+          "publish": false,
+          "snapshotIdentiferTemplate": "https://github.com/ambanum/OpenTermsArchive-snapshots/commit/%SNAPSHOT_ID",
+          "author": {
+            "name": "Open Terms Archive Testing Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        },
+        "mongo": {
+          "connectionURI": "mongodb://127.0.0.1:27017",
+          "database": "open-terms-archive-test",
+          "collection": "versions"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "path": "./test/data/snapshots",
+          "publish": false,
+          "author": {
+            "name": "Open Terms Archive Testing Bot",
+            "email": "bot@opentermsarchive.org"
+          }
+        },
+        "mongo": {
+          "connectionURI": "mongodb://127.0.0.1:27017",
+          "database": "open-terms-archive-test",
+          "collection": "snapshots"
+        }
+      }
+    }
+  },
+  "fetcher": {
+    "waitForElementsTimeout": 1000
+  },
+  "dataset": {
+    "title": "sandbox",
+    "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox"
+  }
+}

package/config/vagrant.json

@@ -0,0 +1,24 @@
+{
+  "services": {
+    "repository": "https://github.com/OpenTermsArchive/contrib-declarations.git"
+  },
+  "recorder": {
+    "versions": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/sandbox-versions.git"
+        }
+      }
+    },
+    "snapshots": {
+      "storage": {
+        "git": {
+          "repository": "git@github.com:OpenTermsArchive/sandbox-snapshots.git"
+        }
+      }
+    }
+  },
+  "logger": {
+    "sendMailOnError": false
+  }
+}

package/decision-records/0001-service-name-and-id.md

@@ -0,0 +1,73 @@
+# Choosing service name and service ID
+
+- Date: 2020-10-14
+
+## Context and Problem Statement
+
+To scale up from 50 to 5,000 services, we need a clear way for choosing the service name and the service ID.
+
+### We need
+
+A name that reflects the common name used by the provider itself, to be exposed in a GUI. This name is currently exposed as the name property in the service declaration.
+An ID of sorts that can be represented in the filesystem. This ID is currently exposed as the filename of the service declaration, without the .json extension.
+
+### Use cases
+
+The service name is presented to end users. It should reflect as closely as possible the official service name, so that it can be identified easily.
+The ID is used internally and exposed for analysis. It should be easy to handle with scripts and other tools.
+
+### Constraints for the ID
+
+As long as this ID is stored in the filesystem:
+
+- No `/` for UNIX.
+- No `\` for Windows.
+- No `:` for APFS and HFS.
+- No case-sensitive duplicates to support case-insensitive filesystems.
+- No more than 255 characters to support transfer over [FAT32](https://en.wikipedia.org/wiki/File_Allocation_Table#FAT32).
+
+UTF, spaces and capitals are all supported, even on case-insensitive filesystems.
+
+### However
+
+- UTF in filenames can be [a (fixable) problem with Git and HFS+](https://stackoverflow.com/questions/5581857/git-and-the-umlaut-problem-on-mac-os-x).
+- UTF in filenames is by default quoted in Git, leading for example `été.txt` to be displayed as `"\303\251t\303\251.txt"`.
+- Most online services align their brand name with their domain name. Even though UTF is now officially supported in domain names, support is limited and most services, even non-Western, have an official ASCII transliteration used at least in their domain name (e.g. “qq” by Tencent, “rzd.ru” for “РЖД”, “yahoo” for “Yahoo!”).
+- We currently use GitHub as a GUI, so the service ID is presented to the user instead of the service name. The name is used in email notifications.
+
+## Decision Outcome
+
+1. The service name should be the one used by the service itself, no matter the alphabet.
+
+- _Example: `туту.ру`_.
+
+2. We don't support non-ASCII characters in service IDs, at least as long as the database is Git and the filesystem, in order to minimise risk. Service IDs are derived from the service name through normalization into ASCII.
+
+- _Example: `туту.ру` → `tutu.ru`_.
+- _Example: `historielærer.dk` → `historielaerer.dk`_.
+- _Example: `RTÉ` → `RTE`_.
+
+3. We support punctuation, except characters that have meaning at filesystem level (`:`, `/`, `\`). These are replaced with a dash (`-`).
+
+- _Example: `Yahoo!` → `Yahoo!`_.
+- _Example: `Last.fm` → `Last.fm`_.
+- _Example: `re:start` → `re-start`_.
+- _Example: `we://` → `we---`_.
+
+4. We support capitals. Casing is expected to reflect the official service name casing.
+
+- _Example: `hi5` → `hi5`_.
+- _Example: `DeviantArt` → `DeviantArt`_.
+- _Example: `LINE` → `LINE`_.
+
+5. We support spaces. Spaces are expected to reflect the official service name spacing.
+
+- _Example: `App Store` → `App Store`_.
+- _Example: `DeviantArt` → `DeviantArt`_.
+
+6. We prefix the service name by the provider name when self-references are ambiguous, separated by a space. For example, Facebook refers to their Self-serve Ads service simply as “Ads”, which we cannot use in a shared database. We thus call the service “Facebook Ads”.
+
+- _Example: `Ads` (by Facebook) → `Facebook Ads`_.
+- _Example: `Analytics` (by Google) → `Google Analytics`_.
+- _Example: `Firebase` (by Google) → `Firebase`_.
+- _Example: `App Store` (by Apple) → `App Store`_.

package/decision-records/0002-service-history.md

@@ -0,0 +1,212 @@
+# Defining a service history system
+
+- Date: 2020-11-23
+
+## Context and Problem Statement
+
+We need to be able to regenerate versions from snapshots. As documents is aim to change over time (location or filters) we can't rely on the last version of the declaration to regenerate the version from an old snapshot. So we need a system to keep track of declaration changes, that's what we called declarations and filters versioning.
+
+## Solutions considered
+
+At this time, we see three solutions which have in common the following rules:
+
+- `history` is optional
+- the current valid declaration has no date and should be clearly identifiable
+- the `valid_until` date is an inclusive expiration date. It should be the exact authored date of the last snapshot commit for which the declaration is still valid.
+
+## Option 1: Add an `history` field in service declaration
+
+In `services/ASKfm.json`:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": ".selection",
+      "filter": [ "add" ]
+      "history": [
+        {
+          "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+          "select": "body",
+          "filter": [ "add" ]
+          "valid_until": "2020-08-24T14:02:39Z"
+        },
+        {
+          "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+          "select": "body",
+          "valid_until": "2020-08-23T14:02:39Z"
+        }
+      ]
+    }
+  }
+}
+```
+
+Note: When no historisation is needed the file may have no mention of history.
+
+**Pros:**
+
+- Everything is in the same file:
+  - might prevent to forget to update existing history
+  - might help user to know that history is a thing and encourage them to learn about it if they feel the need
+  - no (pseudo-)hidden knowledge about history
+
+**Cons:**
+
+- Apparent complexity can discourage new contributors
+- With time, the file can become huge
+
+## Option 2: Add an `serviceId.history.json` file
+
+In `services/ASKfm.json`:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": ".selection",
+      "filter": [ "add" ]
+    }
+  }
+}
+```
+
+In `services/ASKfm.history.json`:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": [
+      {
+        "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+        "select": "body",
+        "filter": [ "add" ]
+        "valid_until": "2020-08-24T14:02:39Z"
+      },
+      {
+        "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+        "select": "body",
+        "valid_until": "2020-08-23T14:02:39Z"
+      }
+    ]
+  }
+}
+```
+
+**Pros:**
+
+- Service declaration stay small and simple
+- History file is kept close to the service declaration so users might see them
+
+**Cons:**
+
+- Make the discovery of history capacities less easy
+- Increase the probability of forgetting to update history file when making a change in the service discovery
+
+## Option 2A
+
+Same as option 2, but the history file should only contain the document declarations to avoid divergence on service properties with the one in the original file.
+
+In `services/ASKfm.json`, **called the “service declaration”**:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": ".selection",
+      "filter": [ "add" ]
+    }
+  }
+}
+```
+
+In `services/ASKfm.history.json`, **called the “service history”**:
+
+```
+{
+  "Terms of Service": [
+    {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": "body",
+      "filter": [ "add" ]
+      "valid_until": "2020-08-24T14:02:39Z"
+    },
+    {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": "body",
+      "valid_until": "2020-08-23T14:02:39Z"
+    }
+  ]
+}
+```
+
+## Option 3: Add an history service declaration file in `services/history` folder
+
+In `services/ASKfm.json`:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": {
+      "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+      "select": ".selection",
+      "filter": [ "add" ]
+    }
+  }
+}
+```
+
+In `services/history/ASKfm.json`:
+
+```
+{
+  "name": "ASKfm",
+  "documents": {
+    "Terms of Service": [
+      {
+        "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+        "select": "body",
+        "filter": [ "add" ]
+        "valid_until": "2020-08-24T14:02:39Z"
+      },
+      {
+        "fetch": "https://ask.fm/docs/terms_of_use/?lang=en",
+        "select": "body",
+        "valid_until": "2020-08-23T14:02:39Z"
+      }
+    ]
+  }
+}
+```
+
+**Pros:**
+
+- Service declaration stay small and simple
+- All history updates are reserved to users with the knowledge that might work as gatekeepers
+
+**Cons:**
+
+- All history updates are reserved to users with the knowledge that might work as gatekeepers :)
+- Need to rely on people with knowledge to keep the history
+
+## Some thoughts
+
+### Community
+
+The choice might have implication on the community that will grow around the project.
+
+_Option 1_ shows everything to everyone, it might frightened some contributors with some apparent complexity (once there are history in the declaration file), but it might also encourage them to learn about it if they want or feel the need to. All contributors will share the same view and knowledge about the system. This might encourage collaboration between them to learn and improve together.
+
+_Option 2_ and _Option 3_ hide the complexity of history management in separate files and only most adventurous contributors will find them by themselves. Contribution to those files will probably be done by specific contributors that will be taught to manage those file. Thus creating two different kind of contributors: those who will stay with the basic service declaration, not knowing that more complex options exist, and those who will have the knowledge of history management whose work might stay in the shadow or work as gatekeeper.
+
+## Decision Outcome
+
+[After consulting the community](https://github.com/ambanum/OpenTermsArchive/issues/156), the options 2A is retained as it hide complexity (compared to Option 1) of the history while increasing its discoverability (compared to Option 3) for contributors who might become more “adventurous”.