@tricoteuses/assemblee 2.4.3 → 2.5.0

package/README.md

@@ -100,9 +100,9 @@ If you use such options, use them in all subsequent commands too (_data:regorgan
 - `--dataset` or `-d <name>`: Clean a specific dataset only
 - `--no-reset-after-commit`: Skip Git reset after committing (useful to preserve local changes)
 - `--no-validate` or `-V`: Skip schema validation during cleaning
-- `--fullCompteRenduCommissions`: Force reprocessing of commission reports
-- `--fetchDocuments` : Specify to retrieve documents like reports, videos metadata files
+- `--fetchDocuments` : Specify to retrieve documents
 - `--parseDocuments`: Specify to parse documents into cleaned json
+- `--fetchVideos`: Retrieve videos
 
 ### Options for Retrieving Documents
 

package/lib/loaders.js

@@ -3,12 +3,15 @@ import i from "fs-extra";
 import n from "node:path";
 import "node:crypto";
 import { L as r, a1 as R, a2 as Z, a3 as q, a4 as B, a5 as J, a6 as N, Y, $ as G, a0 as K, X as ee, Z as te, _ as se } from "./uids-DaRrTkI-.js";
-import { w as z } from "./file_systems-CzTtoLWb.js";
+import { w as P } from "./file_systems-CzTtoLWb.js";
 import { C as w } from "./amendements-40Z7OJLT.js";
-var I = /* @__PURE__ */ ((s) => (s[s.None = 0] = "None", s[s.ActeursEtOrganes = 1] = "ActeursEtOrganes", s[s.Agendas = 2] = "Agendas", s[s.Amendements = 4] = "Amendements", s[s.DossiersLegislatifs = 8] = "DossiersLegislatifs", s[s.Photos = 16] = "Photos", s[s.Scrutins = 32] = "Scrutins", s[s.Questions = 64] = "Questions", s[s.ComptesRendusSeances = 128] = "ComptesRendusSeances", s[s.All = 255] = "All", s))(I || {});
-function y(s, t) {
-  const e = n.join(t, s.filename);
-  i.existsSync(e) && i.removeSync(e), i.existsSync(n.join(t, "json")) ? i.moveSync(n.join(t, "json"), e) : i.mkdirSync(e);
+var I = /* @__PURE__ */ ((t) => (t[t.None = 0] = "None", t[t.ActeursEtOrganes = 1] = "ActeursEtOrganes", t[t.Agendas = 2] = "Agendas", t[t.Amendements = 4] = "Amendements", t[t.DossiersLegislatifs = 8] = "DossiersLegislatifs", t[t.Photos = 16] = "Photos", t[t.Scrutins = 32] = "Scrutins", t[t.Questions = 64] = "Questions", t[t.ComptesRendusSeances = 128] = "ComptesRendusSeances", t[t.All = 255] = "All", t))(I || {});
+function y(t, s) {
+  const e = n.join(s, t.filename);
+  i.existsSync(e) && i.removeSync(e), i.existsSync(n.join(s, `${t.filename}.temp`, "json")) ? i.moveSync(
+    n.join(s, `${t.filename}.temp`, "json"),
+    e
+  ) : i.mkdirSync(e);
 }
 const S = {
   acteursEtOrganes: [
@@ -276,13 +279,15 @@ const S = {
       filename: "AMO40_deputes_actifs_mandats_actifs_organes_divises_XVII.json",
       ignoreForWeb: !0,
       legislature: r.DixSept,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
-        i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.pathExistsSync(n.join(t, "acteur")) && i.moveSync(
-          n.join(t, "acteur"),
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
+        i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.pathExistsSync(
+          n.join(s, `${t.filename}.temp`, "acteur")
+        ) && i.moveSync(
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -318,13 +323,15 @@ const S = {
       filename: "AMO40_deputes_actifs_mandats_actifs_organes_divises_XVI.json",
       ignoreForWeb: !0,
       legislature: r.Seize,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
-        i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.pathExistsSync(n.join(t, "acteur")) && i.moveSync(
-          n.join(t, "acteur"),
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
+        i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.pathExistsSync(
+          n.join(s, `${t.filename}.temp`, "acteur")
+        ) && i.moveSync(
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -360,13 +367,13 @@ const S = {
       filename: "AMO40_deputes_actifs_mandats_actifs_organes_divises_XV.json",
       ignoreForWeb: !0,
       legislature: r.Quinze,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
         i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.moveSync(
-          n.join(t, "acteur"),
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -392,16 +399,16 @@ const S = {
       filename: "AMO50_acteurs_mandats_organes_divises_XVII.json",
       ignoreForWeb: !0,
       legislature: r.DixSept,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
         i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.moveSync(
-          n.join(t, "mandat"),
+          n.join(s, `${t.filename}.temp`, "mandat"),
           n.join(e, "mandat")
         ), i.moveSync(
-          n.join(t, "acteur"),
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -427,16 +434,16 @@ const S = {
       filename: "AMO50_acteurs_mandats_organes_divises_XVI.json",
       ignoreForWeb: !0,
       legislature: r.Seize,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
         i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.moveSync(
-          n.join(t, "mandat"),
+          n.join(s, `${t.filename}.temp`, "mandat"),
           n.join(e, "mandat")
         ), i.moveSync(
-          n.join(t, "acteur"),
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -462,16 +469,16 @@ const S = {
       filename: "AMO50_acteurs_mandats_organes_divises_XV.json",
       ignoreForWeb: !0,
       legislature: r.Quinze,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
         i.existsSync(e) && i.removeSync(e), i.mkdirSync(e), i.moveSync(
-          n.join(t, "mandat"),
+          n.join(s, `${t.filename}.temp`, "mandat"),
           n.join(e, "mandat")
         ), i.moveSync(
-          n.join(t, "acteur"),
+          n.join(s, `${t.filename}.temp`, "acteur"),
           n.join(e, "acteur")
         ), i.moveSync(
-          n.join(t, "organe"),
+          n.join(s, `${t.filename}.temp`, "organe"),
           n.join(e, "organe")
         );
       },
@@ -668,9 +675,12 @@ const S = {
     {
       filename: "Comptes_Rendus_Seances_XVII.xml",
       legislature: r.DixSept,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
-        i.existsSync(e) && i.removeSync(e), i.moveSync(n.join(t, "xml"), e);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
+        i.existsSync(e) && i.removeSync(e), i.moveSync(
+          n.join(s, `${t.filename}.temp`, "xml"),
+          e
+        );
       },
       structure: 1,
       title: "Comptes rendus des séances publiques de la XVIIème législature",
@@ -679,9 +689,12 @@ const S = {
     {
       filename: "Comptes_Rendus_Seances_XVI.xml",
       legislature: r.Seize,
-      repairZip: (s, t) => {
-        const e = n.join(t, s.filename);
-        i.existsSync(e) && i.removeSync(e), i.moveSync(n.join(t, "xml"), e);
+      repairZip: (t, s) => {
+        const e = n.join(s, t.filename);
+        i.existsSync(e) && i.removeSync(e), i.moveSync(
+          n.join(s, `${t.filename}.temp`, "xml"),
+          e
+        );
       },
       structure: 1,
       title: "Comptes rendus des séances publiques de la XVIème législature",
@@ -716,57 +729,57 @@ const S = {
   1,
   "I"
 ];
-function ne(s) {
-  if (s < 0)
+function ne(t) {
+  if (t < 0)
     return;
-  if (s === 0)
+  if (t === 0)
     return "0I";
-  let t = "";
+  let s = "";
   for (let e = 0; e < W.length; e += 2)
-    for (; s >= W[e]; )
-      t += W[e + 1], s -= W[e];
-  return t;
+    for (; t >= W[e]; )
+      s += W[e + 1], t -= W[e];
+  return s;
 }
-function* $(s, t, e, a, l, d = "json", { enriched: _ = !1, log: j = !1, noValidation: h = !1 } = {}) {
+function* z(t, s, e, a, l, d = "json", { enriched: _ = !1, log: j = !1, noValidation: h = !1 } = {}) {
   const F = /* @__PURE__ */ new Set();
   for (const v of e) {
-    if (v.ignoreForWeb || !ie(t, v))
+    if (v.ignoreForWeb || !ie(s, v))
       continue;
-    const b = new RegExp(`\\.${d}$`), P = n.join(
-      s,
+    const b = new RegExp(`\\.${d}$`), x = n.join(
+      t,
       v.filename.replace(
         b,
         _ ? "_enrichi" : "_nettoye"
       )
     );
-    for (const D of z(P, l)) {
+    for (const D of P(x, l)) {
       if (!D[D.length - 1].endsWith(".json"))
         continue;
-      const X = n.join(P, ...D);
+      const X = n.join(x, ...D);
       j && console.log(`Loading file: ${X}…`);
       let V = i.readFileSync(X, { encoding: "utf8" });
-      const L = h ? JSON.parse(V) : a(V);
-      F.has(L.uid) || (F.add(L.uid), yield {
+      const $ = h ? JSON.parse(V) : a(V);
+      F.has($.uid) || (F.add($.uid), yield {
         datasetLegislature: v.legislature,
         filePath: X,
-        item: L
+        item: $
       }, V = null);
     }
   }
   F.clear();
 }
-function ie(s, t) {
-  const e = Array.isArray(s) ? s : s !== void 0 ? [s] : [];
-  return s === void 0 || e.includes(0) || t.legislature === r.All ? !0 : e.includes(t.legislature);
+function ie(t, s) {
+  const e = Array.isArray(t) ? t : t !== void 0 ? [t] : [];
+  return t === void 0 || e.includes(0) || s.legislature === r.All ? !0 : e.includes(s.legislature);
 }
-function* de(s, t, e = {}) {
+function* de(t, s, e = {}) {
   for (const {
     datasetLegislature: a,
     item: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.acteursEtOrganes,
     R.toActeur,
     ["acteurs"],
@@ -775,14 +788,14 @@ function* de(s, t, e = {}) {
   ))
     yield { acteur: l, datasetLegislature: a, filePath: d };
 }
-function* pe(s, t, e = {}) {
+function* pe(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.acteursEtOrganes,
     R.toOrgane,
     ["organes"],
@@ -791,14 +804,14 @@ function* pe(s, t, e = {}) {
   ))
     yield { organe: a, datasetLegislature: l, filePath: d };
 }
-function* ge(s, t, e = {}) {
+function* ge(t, s, e = {}) {
   for (const {
     datasetLegislature: a,
     item: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.amendements,
     w.toAmendement,
     [],
@@ -807,14 +820,14 @@ function* ge(s, t, e = {}) {
   ))
     yield { amendement: l, datasetLegislature: a, filePath: d };
 }
-function* _e(s, t, e = {}) {
+function* _e(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.dossiersLegislatifs,
     q.toDocument,
     ["documents"],
@@ -823,14 +836,14 @@ function* _e(s, t, e = {}) {
   ))
     yield { document: a, datasetLegislature: l, filePath: d };
 }
-function* je(s, t, e = {}) {
+function* je(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.dossiersLegislatifs,
     q.toDossierParlementaire,
     ["dossiers"],
@@ -839,14 +852,14 @@ function* je(s, t, e = {}) {
   ))
     yield { dossierParlementaire: a, datasetLegislature: l, filePath: d };
 }
-function* ye(s, t, e = {}) {
+function* ye(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.agendas,
     Z.toReunion,
     [],
@@ -855,14 +868,14 @@ function* ye(s, t, e = {}) {
   ))
     yield { reunion: a, datasetLegislature: l, filePath: d };
 }
-function* he(s, t, e = {}) {
+function* he(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.scrutins,
     B.toScrutin,
     [],
@@ -871,14 +884,14 @@ function* he(s, t, e = {}) {
   ))
     yield { scrutin: a, datasetLegislature: l, filePath: d };
 }
-function* Se(s, t, e = {}) {
+function* Se(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.questions,
     J.toQuestion,
     [],
@@ -887,14 +900,14 @@ function* Se(s, t, e = {}) {
   ))
     yield { question: a, datasetLegislature: l, filePath: d };
 }
-function* Ae(s, t, e = {}) {
+function* Ae(t, s, e = {}) {
   for (const {
     item: a,
     datasetLegislature: l,
     filePath: d
-  } of $(
-    s,
+  } of z(
     t,
+    s,
     S.comptesRendusSeances,
     N.toCompteRendu,
     [],
@@ -903,39 +916,39 @@ function* Ae(s, t, e = {}) {
   ))
     yield { compteRendu: a, datasetLegislature: l, filePath: d };
 }
-function* Fe(s, t, { enriched: e = !1 } = {}) {
-  const a = `Comptes_Rendus_Commissions_${ne(t)}${e ? "_enrichi" : "_nettoye"}`, l = n.join(s, a);
+function* Fe(t, s, { enriched: e = !1 } = {}) {
+  const a = `Comptes_Rendus_Commissions_${ne(s)}${e ? "_enrichi" : "_nettoye"}`, l = n.join(t, a);
   if (i.existsSync(l))
     for (const d of T(l))
       try {
         const _ = i.readFileSync(d, "utf-8");
         yield {
           compteRendu: JSON.parse(_),
-          datasetLegislature: t,
+          datasetLegislature: s,
           filePath: d
         };
       } catch (_) {
         console.warn(`Erreur lors de la lecture du fichier ${d} :`, _);
       }
 }
-function* T(s) {
-  if (i.existsSync(s))
-    for (const t of i.readdirSync(s, { withFileTypes: !0 })) {
-      const e = n.join(s, t.name);
-      t.isDirectory() ? yield* T(e) : t.isFile() && e.endsWith(".json") && (yield e);
+function* T(t) {
+  if (i.existsSync(t))
+    for (const s of i.readdirSync(t, { withFileTypes: !0 })) {
+      const e = n.join(t, s.name);
+      s.isDirectory() ? yield* T(e) : s.isFile() && e.endsWith(".json") && (yield e);
     }
 }
-function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
+function ve(t, s, e, { enriched: a = !1, log: l = !1 } = {}) {
   const d = {}, _ = {};
-  if (t & I.ActeursEtOrganes)
+  if (s & I.ActeursEtOrganes)
     for (const o of S.acteursEtOrganes) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m, ["acteurs"])) {
+      for (const u of P(m, ["acteurs"])) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -945,7 +958,7 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
         }), f = R.toActeur(A);
         d[f.uid] === void 0 && (d[f.uid] = f);
       }
-      for (const u of z(m, ["organes"])) {
+      for (const u of P(m, ["organes"])) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -957,15 +970,15 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
       }
     }
   const j = {}, h = {}, F = {};
-  if (t & I.Agendas)
+  if (s & I.Agendas)
     for (const o of S.agendas) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m)) {
+      for (const u of P(m)) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -974,9 +987,9 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
           encoding: "utf8"
         }), f = Z.toReunion(A);
         j[f.uid] === void 0 && (j[f.uid] = f);
-        const x = f.timestampDebut.toISOString().split("T")[0];
-        let U = h[x];
-        U === void 0 && (U = h[x] = []), U.push(f);
+        const L = f.timestampDebut.toISOString().split("T")[0];
+        let U = h[L];
+        U === void 0 && (U = h[L] = []), U.push(f);
         const C = f.odj;
         if (C !== void 0)
           for (const H of C.pointsOdj || [])
@@ -987,15 +1000,15 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
       }
     }
   const v = {};
-  if (t & I.Amendements)
+  if (s & I.Amendements)
     for (const o of S.amendements) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m)) {
+      for (const u of P(m)) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -1006,16 +1019,16 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
         v[f.uid] === void 0 && (v[f.uid] = f);
       }
     }
-  const b = {}, P = {}, D = {}, O = {};
-  if (t & I.DossiersLegislatifs)
+  const b = {}, x = {}, D = {}, O = {};
+  if (s & I.DossiersLegislatifs)
     for (const o of S.dossiersLegislatifs) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m, ["documents"])) {
+      for (const u of P(m, ["documents"])) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -1025,22 +1038,22 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
         }), f = q.toDocument(A);
         b[f.uid] === void 0 && (b[f.uid] = f);
       }
-      for (const u of z(m, ["dossiers"])) {
+      for (const u of P(m, ["dossiers"])) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
         l && console.log(`Loading file: ${g}…`);
         const A = i.readFileSync(g, {
           encoding: "utf8"
-        }), f = q.toDossierParlementaire(A), x = f.titreDossier;
-        x.senatChemin && P[x.senatChemin] === void 0 && (P[x.senatChemin] = f), x.titreChemin && D[x.titreChemin] === void 0 && (D[x.titreChemin] = f), O[f.uid] === void 0 && (O[f.uid] = f);
+        }), f = q.toDossierParlementaire(A), L = f.titreDossier;
+        L.senatChemin && x[L.senatChemin] === void 0 && (x[L.senatChemin] = f), L.titreChemin && D[L.titreChemin] === void 0 && (D[L.titreChemin] = f), O[f.uid] === void 0 && (O[f.uid] = f);
       }
     }
   let X = {};
-  if (t & I.Photos) {
+  if (s & I.Photos) {
     {
       const o = n.join(
-        s,
+        t,
         `photos_deputes_${// TODO: Handle Legislature.All.
         e === r.All ? r.DixSept : e}`,
         "deputes.json"
@@ -1053,7 +1066,7 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
     }
     {
       const o = n.join(
-        s,
+        t,
         "photos_senateurs",
         "senateurs.json"
       );
@@ -1065,15 +1078,15 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
     }
   }
   const V = {};
-  if (t & I.Scrutins)
+  if (s & I.Scrutins)
     for (const o of S.scrutins) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m)) {
+      for (const u of P(m)) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -1084,16 +1097,16 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
         V[f.uid] === void 0 && (V[f.uid] = f);
       }
     }
-  const L = {};
-  if (t & I.Questions)
+  const $ = {};
+  if (s & I.Questions)
     for (const o of S.questions) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m)) {
+      for (const u of P(m)) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -1101,19 +1114,19 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
         const A = i.readFileSync(g, {
           encoding: "utf8"
         }), f = J.toQuestion(A);
-        L[f.uid] === void 0 && (L[f.uid] = f);
+        $[f.uid] === void 0 && ($[f.uid] = f);
       }
     }
   const p = {};
-  if (t & I.ComptesRendusSeances)
+  if (s & I.ComptesRendusSeances)
     for (const o of S.comptesRendusSeances) {
       if (o.ignoreForWeb || e !== r.All && o.legislature !== r.All && e !== o.legislature)
         continue;
       const m = n.join(
-        s,
+        t,
         o.filename.replace(/\.json$/, a ? "_enrichi" : "_nettoye")
       );
-      for (const u of z(m)) {
+      for (const u of P(m)) {
         if (!u[u.length - 1].endsWith(".json"))
           continue;
         const g = n.join(m, ...u);
@@ -1128,7 +1141,7 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
     acteurByUid: d,
     amendementByUid: v,
     documentByUid: b,
-    dossierParlementaireBySenatChemin: P,
+    dossierParlementaireBySenatChemin: x,
     dossierParlementaireByTitreChemin: D,
     dossierParlementaireByUid: O,
     organeByUid: _,
@@ -1137,18 +1150,18 @@ function ve(s, t, e, { enriched: a = !1, log: l = !1 } = {}) {
     reunionsByDay: h,
     reunionsByDossierUid: F,
     scrutinByUid: V,
-    questionByUid: L,
+    questionByUid: $,
     compteRenduByUid: p
   };
 }
-function Ie(s, t, e, { log: a = !1 } = {}) {
+function Ie(t, s, e, { log: a = !1 } = {}) {
   const l = {}, d = {};
-  if (t & I.ActeursEtOrganes)
+  if (s & I.ActeursEtOrganes)
     for (const p of S.acteursEtOrganes) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1159,12 +1172,12 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
         d[c.uid] === void 0 && (d[c.uid] = c);
     }
   const _ = {}, j = {}, h = {};
-  if (t & I.Agendas)
+  if (s & I.Agendas)
     for (const p of S.agendas) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1176,20 +1189,20 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
         A === void 0 && (A = j[g] = []), A.push(c);
         const f = c.odj;
         if (f !== void 0)
-          for (const x of f.pointsOdj || [])
-            for (const U of x.dossiersLegislatifsRefs || []) {
+          for (const L of f.pointsOdj || [])
+            for (const U of L.dossiersLegislatifsRefs || []) {
               let C = h[U];
               C === void 0 && (C = h[U] = []), C.push(c);
             }
       }
     }
   const F = {};
-  if (t & I.Amendements)
+  if (s & I.Amendements)
     for (const p of S.amendements) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1198,13 +1211,13 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
         for (const g of c.amendements)
           F[g.uid] === void 0 && (F[g.uid] = g);
     }
-  const v = {}, b = {}, P = {}, D = {};
-  if (t & I.DossiersLegislatifs)
+  const v = {}, b = {}, x = {}, D = {};
+  if (s & I.DossiersLegislatifs)
     for (const p of S.dossiersLegislatifs) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1213,15 +1226,15 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
         v[c.uid] === void 0 && (v[c.uid] = c);
       for (const c of u.dossiersParlementaires) {
         const g = c.titreDossier;
-        g.senatChemin && b[g.senatChemin] === void 0 && (b[g.senatChemin] = c), g.titreChemin && P[g.titreChemin] === void 0 && (P[g.titreChemin] = c), D[c.uid] === void 0 && (D[c.uid] = c);
+        g.senatChemin && b[g.senatChemin] === void 0 && (b[g.senatChemin] = c), g.titreChemin && x[g.titreChemin] === void 0 && (x[g.titreChemin] = c), D[c.uid] === void 0 && (D[c.uid] = c);
       }
     }
   const O = {};
-  if (t & I.Photos) {
+  if (s & I.Photos) {
     const p = {};
     {
       const o = n.join(
-        s,
+        t,
         `photos_deputes_${// TODO: Handle Legislature.All.
         e === r.All ? r.DixSept : e}`,
         "deputes.json"
@@ -1234,7 +1247,7 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
     }
     {
       const o = n.join(
-        s,
+        t,
         "photos_senateurs",
         "senateurs.json"
       );
@@ -1246,12 +1259,12 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
     }
   }
   const X = {};
-  if (t & I.Scrutins)
+  if (s & I.Scrutins)
     for (const p of S.scrutins) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1260,12 +1273,12 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
         X[c.uid] === void 0 && (X[c.uid] = c);
     }
   const V = {};
-  if (t & I.Questions)
+  if (s & I.Questions)
     for (const p of S.questions) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
@@ -1273,26 +1286,26 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
       for (const c of u.questions)
         V[c.uid] === void 0 && (V[c.uid] = c);
     }
-  const L = {};
-  if (t & I.ComptesRendusSeances)
+  const $ = {};
+  if (s & I.ComptesRendusSeances)
     for (const p of S.comptesRendusSeances) {
       if (p.ignoreForWeb || e !== r.All && p.legislature !== r.All && e !== p.legislature)
         continue;
       const o = n.join(
-        s,
+        t,
         p.filename.replace(/\.json$/, "_nettoye.json")
       );
       a && console.log(`Loading file: ${o}`);
       const m = i.readFileSync(o, { encoding: "utf8" }), u = N.toComptesRendus(m);
       for (const c of u.comptesRendus)
-        L[c.uid] === void 0 && (L[c.uid] = c);
+        $[c.uid] === void 0 && ($[c.uid] = c);
     }
   return {
     acteurByUid: l,
     amendementByUid: F,
     documentByUid: v,
     dossierParlementaireBySenatChemin: b,
-    dossierParlementaireByTitreChemin: P,
+    dossierParlementaireByTitreChemin: x,
     dossierParlementaireByUid: D,
     organeByUid: d,
     photoByUid: O,
@@ -1301,11 +1314,11 @@ function Ie(s, t, e, { log: a = !1 } = {}) {
     reunionsByDossierUid: h,
     scrutinByUid: X,
     questionByUid: V,
-    compteRenduByUid: L
+    compteRenduByUid: $
   };
 }
-function be(s, t, e) {
-  const a = n.join(s, "Documents"), l = oe(a, t), d = n.join(l, "index.json");
+function be(t, s, e) {
+  const a = n.join(t, "Documents"), l = oe(a, s), d = n.join(l, "index.json");
   if (!i.pathExistsSync(d))
     return;
   const _ = i.readJsonSync(d);
@@ -1314,36 +1327,36 @@ function be(s, t, e) {
       if (h !== void 0)
         return { format: j, path: n.join(l, h), url: F };
 }
-function De(s, t, { log: e = !1 } = {}) {
+function De(t, s, { log: e = !1 } = {}) {
   const a = n.join(
-    s,
+    t,
     `photos_deputes_${// TODO: Handle Legislature.All.
-    t === r.All ? r.DixSept : t}`,
+    s === r.All ? r.DixSept : s}`,
     "deputes.json"
   );
   e && console.log(`Loading file: ${a}`);
   const l = i.readFileSync(a, { encoding: "utf8" });
   return JSON.parse(l);
 }
-function Xe(s, { log: t = !1 } = {}) {
+function Xe(t, { log: s = !1 } = {}) {
   const e = n.join(
-    s,
+    t,
     "photos_senateurs",
     "senateurs.json"
   );
-  t && console.log(`Loading file: ${e}`);
+  s && console.log(`Loading file: ${e}`);
   const a = i.readFileSync(e, { encoding: "utf8" });
   return JSON.parse(a);
 }
-function oe(s, t) {
-  const e = t.match(Y);
+function oe(t, s) {
+  const e = s.match(Y);
   Q.notStrictEqual(
     e,
     null,
-    `Unexpected structure for document UID: ${t}`
+    `Unexpected structure for document UID: ${s}`
   );
   const a = e.groups, l = n.join(
-    s,
+    t,
     a.documentType ?? a.documentType2
   ), d = n.join(l, a.chambre ?? a.chambre2), _ = n.join(
     d,
@@ -1353,70 +1366,70 @@ function oe(s, t) {
     a.legislature ?? a.legislature2
   ), h = a.etat;
   if (h === void 0)
-    return i.ensureDirSync(j), n.join(j, t);
+    return i.ensureDirSync(j), n.join(j, s);
   const F = n.join(j, h);
   let v = "000000" + (a.numero ?? a.numero2);
   v = v.substring(v.length - 6);
   const b = n.join(F, v.substring(0, 3));
-  return i.ensureDirSync(b), n.join(b, t);
+  return i.ensureDirSync(b), n.join(b, s);
 }
-function Ve(s, t) {
-  return E(s, t, te);
+function Ve(t, s) {
+  return E(t, s, te);
 }
-function Le(s, t) {
-  const e = G.exec(t);
+function $e(t, s) {
+  const e = G.exec(s);
   Q.notStrictEqual(
     e,
     null,
-    `Unexpected structure for reunion UID: ${t}`
+    `Unexpected structure for reunion UID: ${s}`
   );
-  const a = n.join(s, e[1]), l = n.join(a, e[2]), d = n.join(l, e[3]), _ = n.join(d, e[4]), j = n.join(_, e[5]);
+  const a = n.join(t, e[1]), l = n.join(a, e[2]), d = n.join(l, e[3]), _ = n.join(d, e[4]), j = n.join(_, e[5]);
   let h = "000000000" + e[6];
   h = h.substring(h.length - 9);
   const F = n.join(j, h.substring(0, 3)), v = n.join(F, h.substring(3, 3));
-  return i.ensureDirSync(v), n.join(v, t);
+  return i.ensureDirSync(v), n.join(v, s);
 }
-function xe(s, t) {
-  const e = K.exec(t);
+function Le(t, s) {
+  const e = K.exec(s);
   Q.notStrictEqual(
     e,
     null,
-    `Unexpected structure for scrutin UID: ${t}`
+    `Unexpected structure for scrutin UID: ${s}`
   );
-  const a = n.join(s, e[1]), l = n.join(a, e[2]), d = n.join(l, e[3]);
+  const a = n.join(t, e[1]), l = n.join(a, e[2]), d = n.join(l, e[3]);
   let _ = "000000" + e[4];
   _ = _.substring(_.length - 6);
   const j = n.join(d, _.substring(0, 3));
-  return i.ensureDirSync(j), n.join(j, t);
+  return i.ensureDirSync(j), n.join(j, s);
 }
-function Pe(s, t) {
-  return E(s, t, se);
+function xe(t, s) {
+  return E(t, s, se);
 }
-function ze(s, t) {
-  const e = ee.exec(t);
+function Pe(t, s) {
+  const e = ee.exec(s);
   Q.notStrictEqual(
     e,
     null,
-    `Unexpected structure for object with UID: ${t}`
+    `Unexpected structure for object with UID: ${s}`
   );
-  const a = n.join(s, e[2]), l = n.join(a, e[3]), d = n.join(l, e[4]), _ = n.join(d, e[5]);
+  const a = n.join(t, e[2]), l = n.join(a, e[3]), d = n.join(l, e[4]), _ = n.join(d, e[5]);
   let j = "000000000" + e[7];
   j = j.substring(j.length - 9);
   const h = n.join(_, j.substring(0, 3)), F = n.join(h, j.substring(3, 3));
-  return i.ensureDirSync(F), n.join(F, t);
+  return i.ensureDirSync(F), n.join(F, s);
 }
-function E(s, t, e) {
-  const a = e.exec(t);
+function E(t, s, e) {
+  const a = e.exec(s);
   Q.notStrictEqual(
     a,
     null,
-    `Unexpected structure for object with UID: ${t}`
+    `Unexpected structure for object with UID: ${s}`
   );
-  const l = n.join(s, a[1]), d = n.join(l, a[2]);
+  const l = n.join(t, a[1]), d = n.join(l, a[2]);
   let _ = "000000" + a[3];
   _ = _.substring(_.length - 6);
   const j = n.join(d, _.substring(0, 3));
-  return i.ensureDirSync(j), n.join(j, t);
+  return i.ensureDirSync(j), n.join(j, s);
 }
 export {
   I as EnabledDatasets,
@@ -1435,11 +1448,11 @@ export {
   be as loadAssembleeDocumentFile,
   De as loadAssembleePhotosDeputes,
   Xe as loadAssembleePhotosSenateurs,
-  ze as pathFromCompteRenduUid,
+  Pe as pathFromCompteRenduUid,
   oe as pathFromDocumentUid,
   Ve as pathFromDossierParlementaireUid,
-  Pe as pathFromQuestionUid,
-  Le as pathFromReunionUid,
-  xe as pathFromScrutinUid,
-  z as walkDir
+  xe as pathFromQuestionUid,
+  $e as pathFromReunionUid,
+  Le as pathFromScrutinUid,
+  P as walkDir
 };

package/lib/parsers.js

@@ -2023,7 +2023,7 @@ function bs(e, u) {
 var Fu;
 const Ke = (
   // @ts-expect-error `hasOwn` is a standard object method
-  (Fu = Object.hasOwn) !== null && Fu !== void 0 ? Fu : ((e, u) => Object.prototype.hasOwnProperty.call(e, u))
+  (Fu = Object.hasOwn) !== null && Fu !== void 0 ? Fu : (e, u) => Object.prototype.hasOwnProperty.call(e, u)
 ), je = /\s+/, Zu = "data-", gt = /^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i, Es = /^{[^]*}$|^\[[^]*]$/;
 function Tu(e, u, t) {
   var a;

package/lib/scripts/process_open_dataset.d.ts

@@ -1,5 +1,6 @@
+import { Dataset } from '../datasets';
 type ProcessDatasetArguments = {
-    dataset: any;
+    dataset: Dataset;
     dataDir: string;
     options: any;
 };

package/lib/scripts/retrieve_documents.d.ts

@@ -5,7 +5,8 @@ type CommonOptions = {
     legislature: number;
     fetchDocuments?: boolean;
     parseDocuments?: boolean;
-    fullCompteRenduCommissions?: boolean;
+    fetchVideos?: boolean;
+    onlyRecent?: number;
 };
 export declare function downloadAndParse(document: Document, datasetDir: string, options: any): Promise<void>;
 export declare function downloadAndParseVideosAndCR(reunion: Reunion, datasetCleanDir: string, options: CommonOptions): Promise<void>;

package/lib/scripts/retrieve_videos.d.ts

@@ -2,9 +2,7 @@ import { Reunion } from '../types/agendas';
 type CommonOptions = {
     dataDir?: string;
     legislature: number;
-    fetchDocuments?: boolean;
-    parseDocuments?: boolean;
-    fullCompteRenduCommissions?: boolean;
+    "only-recent"?: number;
 };
 export declare function downloadAndParseVideos(reunion: Reunion, datasetCleanDir: string, options: CommonOptions): Promise<void>;
 /**

package/lib/scripts/shared/cli_helpers.d.ts

@@ -60,6 +60,11 @@ export declare const notFoundOption: {
     name: string;
     type: BooleanConstructor;
 };
+export declare const onlyRecentOption: {
+    help: string;
+    name: string;
+    type: NumberConstructor;
+};
 export declare const cloneOption: {
     alias: string;
     help: string;
@@ -85,7 +90,7 @@ export declare const pullOption: {
     name: string;
     type: BooleanConstructor;
 };
-export declare const fullCompteRenduCommissionsOption: {
+export declare const fetchVideosOption: {
     help: string;
     name: string;
     type: BooleanConstructor;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/assemblee",
-  "version": "2.4.3",
+  "version": "2.5.0",
   "description": "Retrieve, clean up & handle French Assemblée nationale's open data",
   "keywords": [
     "Assemblée nationale",
@@ -94,6 +94,7 @@
     "js-yaml": "^4.0.0",
     "node-html-parser": "^7.0.1",
     "node-stream-zip": "^1.11.6",
+    "p-limit": "^7.2.0",
     "sharp": "^0.34.2"
   },
   "devDependencies": {