@socialgouv/cdtn-elasticsearch 1.8.9 → 1.11.1

package/.gitlab-ci.yml

@@ -6,7 +6,7 @@
   extends:
     - .autodevops_install
     - .cdtn_base_rules
-  image: node:14.17.3-alpine3.11
+  image: node:14.18.0-alpine3.11
   needs:
     - job: 📦 logger
       artifacts: true

package/package.json

@@ -1,14 +1,14 @@
 {
   "name": "@socialgouv/cdtn-elasticsearch",
   "description": "SocialGouv - Code du travail numerique - Infrastructure - Elasticsearch",
-  "version": "1.8.9",
+  "version": "1.11.1",
   "babel": {
     "plugins": [
       "@babel/plugin-transform-modules-commonjs"
     ]
   },
   "dependencies": {
-    "@socialgouv/cdtn-logger": "^1.8.9",
+    "@socialgouv/cdtn-logger": "^1.11.1",
     "got": "^11.8.2"
   },
   "license": "Apache-2.0",
@@ -25,12 +25,12 @@
     "url": "https://github.com/SocialGouv/cdtn-admin.git"
   },
   "devDependencies": {
-    "@babel/core": "^7.14.8",
-    "@babel/plugin-transform-modules-commonjs": "^7.14.5",
-    "@socialgouv/eslint-config-recommended": "^1.75.0",
-    "eslint": "^7.30.0",
-    "jest": "^27.0.6",
-    "lint-staged": "^11.0.1",
+    "@babel/core": "^7.15.5",
+    "@babel/plugin-transform-modules-commonjs": "^7.15.4",
+    "@socialgouv/eslint-config-recommended": "^1.88.0",
+    "eslint": "^7.32.0",
+    "jest": "^27.1.1",
+    "lint-staged": "^11.1.2",
     "prettier": "^2.3.2"
   },
   "scripts": {
@@ -46,5 +46,5 @@
   },
   "sideEffects": false,
   "typings": "src/index.d.ts",
-  "gitHead": "529d1bc3ec97f443b7ec21e6b1cbf8ff3fe811b5"
+  "gitHead": "76faf7fa74ef832247ac83bb7201a9310c13099a"
 }

package/src/dataset/stop_words/stop_words.json

@@ -184,7 +184,6 @@
   "envers",
   "environ",
   "es",
-  "essai",
   "est",
   "et",
   "etant",

package/src/mapping/document.mapping.js

@@ -83,6 +83,9 @@ exports.documentMapping = {
 
     folder: { type: "text" },
 
+    // html view for all documents
+    html: { type: "text" },
+
     // available for themes
     icon: { type: "keyword" },
 
@@ -131,6 +134,9 @@ exports.documentMapping = {
 
     publishedAt: { type: "date" },
 
+    // XML in JSON (available for fiche SP)
+    raw: { type: "text" },
+
     // available for themes and highlights
     refs: {
       properties: {

package/src/vectorizer/index.js

@@ -32,19 +32,17 @@ function preprocess(text) {
 }
 
 async function callTFServe(json) {
-  const { body } = await got.post(tfServeURL, {
+  const response = await got.post(tfServeURL, {
     cache,
     json,
     responseType: "json",
-    retry: {
-      limit: 15,
-      methods: ["POST"],
-    },
+    retries: 0,
+    timeout: 600000,
   });
-  return body["outputs"];
+  return response.body["outputs"];
 }
 
-async function vectorizeDocument(title, content) {
+async function vectorizeDocument(id, title, content) {
   if (title == undefined || title == "") {
     throw new Error("Cannot vectorize document with empty title.");
   }
@@ -56,7 +54,10 @@ async function vectorizeDocument(title, content) {
     inputs: { context, input },
     signature_name: "response_encoder",
   };
+  console.time(`CALL TF SERVER ${id}`);
   const vectors = await callTFServe(body);
+  console.timeEnd(`CALL TF SERVER ${id}`);
+
   return vectors[0];
 }
 
@@ -71,6 +72,7 @@ async function vectorizeQuery(query) {
     signature_name: "question_encoder",
   };
   const vectors = await callTFServe(body);
+  console.log("vectors lenght", vectors.length);
   return vectors[0];
 }
 

package/src/vectorizer/index.test.js

@@ -5,12 +5,12 @@ const timeout = 10000;
 test(
   "Should vectorize document",
   async () => {
-    const vector1 = await vectorizeDocument("titre", "contenu");
+    const vector1 = await vectorizeDocument("id", "titre", "contenu");
     expect(vector1).toBeDefined();
     expect(vector1).toMatchSnapshot();
 
     // preprocessing should make those embeddings equal
-    const vector2 = await vectorizeDocument("le titre", "et le contènu");
+    const vector2 = await vectorizeDocument("id", "le titre", "et le contènu");
     expect(vector2).toEqual(vector1);
   },
   timeout