@tricoteuses/senat 1.1.2 → 1.3.1

package/lib/scripts/convert_data.mjs

@@ -1,116 +1,89 @@
-import fs from 'fs-extra';
-import path from 'path';
-import commandLineArgs from 'command-line-args';
-import { datasets, EnabledDatasets, getEnabledDatasets, } from '../datasets';
-import { Aggregator, allFollows, getAllLois, getAllQuestions, getAllSens } from '../model';
-import { insertLoiReferences, insertQuestionReferences, insertSenReferences } from '../inserters';
+import assert from "assert";
+import commandLineArgs from "command-line-args";
+import fs from "fs-extra";
+import path from "path";
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets";
+import { DOCUMENT_METADATA_FILE, DOSLEG_DOSSIERS_FOLDER, RAPPORT_FOLDER, SENS_CIRCONSCRIPTIONS_FOLDER, SENS_ORGANISMES_FOLDER, SENS_SENATEURS_FOLDER, TEXTE_FOLDER, TEXTE_ORIGINAL_FOLDER, } from "../loaders";
+import { findAllAmendements, findAllCirconscriptions, findAllLois, findAllOrganismes, findAllQuestions, findAllSens, } from "../model";
+import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg";
+import { formatToFourDigitSession, SIGNET_STRUCTURE_REGEXP, UNDEFINED_SESSION } from "./datautil";
+import { commonOptions } from "./shared/cli_helpers";
+import { ensureAndClearDir } from "./shared/util";
 const optionsDefinitions = [
-    {
-        alias: 'k',
-        defaultValue: ['All'],
-        help: 'categories of datasets to reorganize',
-        multiple: true,
-        name: 'categories',
-        type: String,
-    },
-    {
-        alias: 's',
-        help: 'don\'t log anything',
-        name: 'silent',
-        type: Boolean,
-    },
-    {
-        alias: "v",
-        help: "verbose logs",
-        name: "verbose",
-        type: Boolean,
-    },
-    {
-        defaultOption: true,
-        help: 'directory containing Sénat open data files',
-        name: 'dataDir',
-        type: String,
-    },
+    ...commonOptions,
 ];
 const options = commandLineArgs(optionsDefinitions);
-const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
-function ensureAndClearDir(path) {
-    if (!fs.existsSync(path)) {
-        fs.mkdirSync(path);
-    }
-    else {
-        fs.emptyDirSync(path);
-    }
-}
+const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/";
+const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/";
+const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/";
+const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/";
 async function convertData() {
     const enabledDatasets = getEnabledDatasets(options.categories);
     const dataDir = options.dataDir;
-    if (enabledDatasets & EnabledDatasets.DosLeg) {
-        const dataset = datasets.dosleg;
+    assert(dataDir, "Missing argument: data directory");
+    console.time("data transformation time");
+    if (enabledDatasets & EnabledDatasets.Ameli) {
+        const dataset = datasets.ameli;
         if (!options.silent) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const lois = await getAllLois();
-        const aggregator = new Aggregator(new Set(allFollows));
-        for (const loi of lois) {
-            aggregator.addLoi(loi);
-        }
-        if (options.verbose) {
-            console.log("Starting getting all the data…");
-            console.time("get all data");
+        const ameliReorganizedRootDir = path.join(dataDir, dataset.database);
+        ensureAndClearDir(ameliReorganizedRootDir);
+        for await (const amendement of findAllAmendements()) {
+            if (options.verbose) {
+                console.log(`Converting ${amendement.numero} file…`);
+            }
+            const session = String(amendement.session) || UNDEFINED_SESSION;
+            const signetDossierLegislatif = amendement.signet_dossier_legislatif
+                || `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase();
+            const ameliReorganizedDir = path.join(ameliReorganizedRootDir, session, signetDossierLegislatif);
+            fs.ensureDirSync(ameliReorganizedDir);
+            const amendementFileName = `${amendement.numero}.json`;
+            fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 });
         }
-        await aggregator.getAll();
-        if (options.verbose) {
-            console.timeEnd("get all data");
+    }
+    if (enabledDatasets & EnabledDatasets.DosLeg) {
+        const dataset = datasets.dosleg;
+        if (!options.silent) {
+            console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const allData = aggregator.toJson();
-        const loiReorganizedRootDir = path.join(dataDir, dataset.database);
-        ensureAndClearDir(loiReorganizedRootDir);
-        for (const loi of lois) {
+        const doslegReorganizedRootDir = path.join(dataDir, dataset.database);
+        const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER);
+        ensureAndClearDir(doslegReorganizedRootDir);
+        ensureAndClearDir(dossiersReorganizedDir);
+        for await (const loi of findAllLois()) {
             if (options.verbose) {
                 console.log(`Converting ${loi.signet} file…`);
             }
-            insertLoiReferences(loi, allData, {});
-            let loiFileName = `${loi.loicod}.json`;
-            let loiReorganizedDir = path.join(loiReorganizedRootDir, loi.typloicod);
-            if (loi.signet) {
-                loiFileName = `${loi.signet}.json`;
-                let signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups;
-                if (signetParts) {
-                    const { session } = signetParts;
-                    loiReorganizedDir = path.join(loiReorganizedDir, session);
-                }
-                else {
-                    loiReorganizedDir = path.join(loiReorganizedDir, "non defini");
-                }
-                fs.ensureDirSync(loiReorganizedDir);
+            let loiReorganizedDir = path.join(dossiersReorganizedDir, UNDEFINED_SESSION);
+            const signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups;
+            if (signetParts) {
+                const { session } = signetParts;
+                const formattedSession = formatToFourDigitSession(session);
+                loiReorganizedDir = path.join(dossiersReorganizedDir, formattedSession);
             }
+            fs.ensureDirSync(loiReorganizedDir);
+            const loiFileName = `${loi.signet}.json`;
             fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, { spaces: 2 });
         }
+        await convertTexteUrls(dataDir);
+        await convertRapportUrls(dataDir);
     }
     if (enabledDatasets & EnabledDatasets.Questions) {
         const dataset = datasets.questions;
         if (!options.silent) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const questions = await getAllQuestions();
-        const aggregator = new Aggregator(new Set(allFollows));
-        for (const question of questions) {
-            aggregator.addQuestion(question);
-        }
-        await aggregator.getAll();
-        const allData = aggregator.toJson();
-        const questionReorganizedRootDir = path.join(dataDir, dataset.database);
-        ensureAndClearDir(questionReorganizedRootDir);
-        for (const question of questions) {
+        const questionsReorganizedRootDir = path.join(dataDir, dataset.database);
+        ensureAndClearDir(questionsReorganizedRootDir);
+        for await (const question of findAllQuestions()) {
             if (options.verbose) {
-                console.log(`Converting ${question.id} file…`);
+                console.log(`Converting ${question.reference} file…`);
             }
-            insertQuestionReferences(question, allData, {});
             const legislature = question.legislature ? question.legislature : 0;
-            const questionReorganizedDir = path.join(questionReorganizedRootDir, String(legislature));
+            const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature));
             fs.ensureDirSync(questionReorganizedDir);
-            let questionFileName = `${question.reference}.json`;
+            const questionFileName = `${question.reference}.json`;
             fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
         }
     }
@@ -119,24 +92,86 @@ async function convertData() {
         if (!options.silent) {
             console.log(`Converting database ${dataset.database} data into files…`);
         }
-        const sens = await getAllSens();
-        const aggregator = new Aggregator(new Set(allFollows));
-        for (const sen of sens) {
-            aggregator.addSen(sen);
+        const sensReorganizedRootDir = path.join(dataDir, dataset.database);
+        const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER);
+        const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER);
+        const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER);
+        ensureAndClearDir(sensReorganizedRootDir);
+        ensureAndClearDir(senateursReorganizedDir);
+        ensureAndClearDir(circonscriptionsReorganizedDir);
+        ensureAndClearDir(organismesReorganizedDir);
+        for await (const sen of findAllSens()) {
+            if (options.verbose) {
+                console.log(`Converting ${sen.matricule} file…`);
+            }
+            const senFileName = `${sen.matricule}.json`;
+            fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, { spaces: 2 });
+        }
+        for await (const circonscription of findAllCirconscriptions()) {
+            if (options.verbose) {
+                console.log(`Converting ${circonscription.identifiant} file…`);
+            }
+            const circonscriptionFileName = `${circonscription.identifiant}.json`;
+            fs.writeJSONSync(path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 });
         }
-        await aggregator.getAll();
-        const allData = aggregator.toJson();
-        const senReorganizedRootDir = path.join(dataDir, dataset.database);
-        ensureAndClearDir(senReorganizedRootDir);
-        for (const sen of sens) {
+        for await (const organisme of findAllOrganismes()) {
             if (options.verbose) {
-                console.log(`Converting ${sen.senmat} file…`);
+                console.log(`Converting ${organisme.code} file…`);
             }
-            insertSenReferences(sen, allData, {});
-            let senFileName = `${sen.senmat}.json`;
-            fs.writeJSONSync(path.join(senReorganizedRootDir, senFileName), sen, { spaces: 2 });
+            const organismeFileName = `${organisme.code}.json`;
+            fs.writeJSONSync(path.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 });
         }
     }
+    if (!options.silent) {
+        console.timeEnd("data transformation time");
+    }
+}
+async function convertTexteUrls(dataDir) {
+    const textesDir = path.join(dataDir, TEXTE_FOLDER);
+    fs.ensureDirSync(textesDir);
+    const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER);
+    for await (const texte of findSenatTexteUrls(options.sessions)) {
+        const texteName = path.parse(texte.url).name;
+        const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName);
+        fs.ensureDirSync(texteDir);
+        const metadata = {
+            name: texteName,
+            session: texte.session,
+            url_expose_des_motifs: texte.hasExposeDesMotifs ?
+                new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL) : undefined,
+            url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
+            url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
+            url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
+        };
+        fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 });
+    }
+}
+async function convertRapportUrls(dataDir) {
+    const rapportsDir = path.join(dataDir, RAPPORT_FOLDER);
+    fs.ensureDirSync(rapportsDir);
+    for await (const rapport of findSenatRapportUrls(options.sessions)) {
+        const parsedRapportUrl = path.parse(rapport.url);
+        const rapportName = parsedRapportUrl.name;
+        const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName);
+        fs.ensureDirSync(rapportDir);
+        const rapportHtmlUrlBase = `${rapportName}_mono.html`;
+        const rapportHtmlUrl = path.format({
+            dir: parsedRapportUrl.dir,
+            base: rapportHtmlUrlBase,
+        });
+        const rapportPdfUrlBase = `${rapportName}1.pdf`;
+        const rapportPdfUrl = path.format({
+            dir: parsedRapportUrl.dir,
+            base: rapportPdfUrlBase,
+        });
+        const metadata = {
+            name: rapportName,
+            session: rapport.session,
+            url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
+            url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
+        };
+        fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 });
+    }
 }
 convertData()
     .then(() => process.exit(0))

package/lib/scripts/convert_data.ts

@@ -1,113 +1,108 @@
-import fs from 'fs-extra'
-import path from 'path'
-import commandLineArgs from 'command-line-args'
+import assert from "assert"
+import commandLineArgs from "command-line-args"
+import fs from "fs-extra"
+import path from "path"
 
-import { datasets, EnabledDatasets, getEnabledDatasets, } from '../datasets'
+import { datasets, EnabledDatasets, getEnabledDatasets } from "../datasets"
 import {
-  Aggregator,
-  allFollows,
-  getAllLois,
-  getAllQuestions,
-  getAllSens
-} from '../model'
+  DOCUMENT_METADATA_FILE,
+  DOSLEG_DOSSIERS_FOLDER,
+  RAPPORT_FOLDER,
+  RapportMetadata,
+  SENS_CIRCONSCRIPTIONS_FOLDER,
+  SENS_ORGANISMES_FOLDER,
+  SENS_SENATEURS_FOLDER,
+  TEXTE_FOLDER,
+  TEXTE_ORIGINAL_FOLDER,
+  TexteMetadata,
+} from "../loaders"
 import {
-  insertLoiReferences,
-  insertQuestionReferences,
-  insertSenReferences
-} from '../inserters'
+  findAllAmendements,
+  findAllCirconscriptions,
+  findAllLois,
+  findAllOrganismes,
+  findAllQuestions,
+  findAllSens,
+} from "../model"
+import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg"
+import { formatToFourDigitSession, SIGNET_STRUCTURE_REGEXP, UNDEFINED_SESSION } from "./datautil"
+import { commonOptions } from "./shared/cli_helpers"
+import { ensureAndClearDir } from "./shared/util"
 
 const optionsDefinitions = [
-  {
-    alias: 'k',
-    defaultValue: ['All'],
-    help: 'categories of datasets to reorganize',
-    multiple: true,
-    name: 'categories',
-    type: String,
-  },
-  {
-    alias: 's',
-    help: 'don\'t log anything',
-    name: 'silent',
-    type: Boolean,
-  },
-  {
-    alias: "v",
-    help: "verbose logs",
-    name: "verbose",
-    type: Boolean,
-  },
-  {
-    defaultOption: true,
-    help: 'directory containing Sénat open data files',
-    name: 'dataDir',
-    type: String,
-  },
+  ...commonOptions,
 ]
 const options = commandLineArgs(optionsDefinitions)
 
-const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/
+const SENAT_TEXTE_XML_BASE_URL = "https://www.senat.fr/akomantoso/"
+const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/leg/"
+const SENAT_EXPOSE_DES_MOTIFS_BASE_URL = "https://www.senat.fr/leg/exposes-des-motifs/"
+const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/"
 
-function ensureAndClearDir(path: string) {
-  if (!fs.existsSync(path)) {
-    fs.mkdirSync(path)
-  } else {
-    fs.emptyDirSync(path)
-  }
-}
-
-async function convertData() {
+async function convertData () {
   const enabledDatasets = getEnabledDatasets(options.categories)
   const dataDir: string = options.dataDir
+  assert(dataDir, "Missing argument: data directory")
 
-  if (enabledDatasets & EnabledDatasets.DosLeg) {
-    const dataset = datasets.dosleg
+  console.time("data transformation time")
+
+  if (enabledDatasets & EnabledDatasets.Ameli) {
+    const dataset = datasets.ameli
 
     if (!options.silent) {
       console.log(`Converting database ${dataset.database} data into files…`)
     }
 
-    const lois = await getAllLois()
-    const aggregator = new Aggregator(new Set(allFollows))
-    for (const loi of lois) {
-      aggregator.addLoi(loi)
-    }
-    if (options.verbose) {
-      console.log("Starting getting all the data…")
-      console.time("get all data")
+    const ameliReorganizedRootDir = path.join(dataDir, dataset.database)
+    ensureAndClearDir(ameliReorganizedRootDir)
+
+    for await (const amendement of findAllAmendements()) {
+      if (options.verbose) {
+        console.log(`Converting ${amendement.numero} file…`)
+      }
+
+      const session = String(amendement.session) || UNDEFINED_SESSION
+      const signetDossierLegislatif = amendement.signet_dossier_legislatif
+        || `${amendement.nature_texte}-${amendement.numero_texte}`.toLowerCase()
+      const ameliReorganizedDir = path.join(ameliReorganizedRootDir, session, signetDossierLegislatif)
+      fs.ensureDirSync(ameliReorganizedDir)
+      const amendementFileName = `${amendement.numero}.json`
+      fs.writeJSONSync(path.join(ameliReorganizedDir, amendementFileName), amendement, { spaces: 2 })
     }
-    await aggregator.getAll()
-    if (options.verbose) {
-      console.timeEnd("get all data")
+  }
+
+  if (enabledDatasets & EnabledDatasets.DosLeg) {
+    const dataset = datasets.dosleg
+
+    if (!options.silent) {
+      console.log(`Converting database ${dataset.database} data into files…`)
     }
-    const allData = aggregator.toJson()
 
-    const loiReorganizedRootDir = path.join(dataDir, dataset.database)
-    ensureAndClearDir(loiReorganizedRootDir)
+    const doslegReorganizedRootDir = path.join(dataDir, dataset.database)
+    const dossiersReorganizedDir = path.join(doslegReorganizedRootDir, DOSLEG_DOSSIERS_FOLDER)
+    ensureAndClearDir(doslegReorganizedRootDir)
+    ensureAndClearDir(dossiersReorganizedDir)
 
-    for (const loi of lois) {
+    for await (const loi of findAllLois()) {
       if (options.verbose) {
         console.log(`Converting ${loi.signet} file…`)
       }
 
-      insertLoiReferences(loi, allData, {})
-
-      let loiFileName = `${loi.loicod}.json`
-      let loiReorganizedDir = path.join(loiReorganizedRootDir, loi.typloicod)
-      if (loi.signet) {
-        loiFileName = `${loi.signet}.json`
-        let signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups
-        if (signetParts) {
-          const { session} = signetParts
-          loiReorganizedDir = path.join(loiReorganizedDir, session)
-        } else {
-          loiReorganizedDir = path.join(loiReorganizedDir, "non defini")
-        }
-        fs.ensureDirSync(loiReorganizedDir)
+      let loiReorganizedDir = path.join(dossiersReorganizedDir, UNDEFINED_SESSION)
+      const signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups
+      if (signetParts) {
+        const { session } = signetParts
+        const formattedSession = formatToFourDigitSession(session)
+        loiReorganizedDir = path.join(dossiersReorganizedDir, formattedSession)
       }
+      fs.ensureDirSync(loiReorganizedDir)
 
-      fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, {spaces: 2})
+      const loiFileName = `${loi.signet}.json`
+      fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, { spaces: 2 })
     }
+
+    await convertTexteUrls(dataDir)
+    await convertRapportUrls(dataDir)
   }
 
   if (enabledDatasets & EnabledDatasets.Questions) {
@@ -117,29 +112,19 @@ async function convertData() {
       console.log(`Converting database ${dataset.database} data into files…`)
     }
 
-    const questions = await getAllQuestions()
-    const aggregator = new Aggregator(new Set(allFollows))
-    for (const question of questions) {
-      aggregator.addQuestion(question)
-    }
-    await aggregator.getAll()
-    const allData = aggregator.toJson()
+    const questionsReorganizedRootDir = path.join(dataDir, dataset.database)
+    ensureAndClearDir(questionsReorganizedRootDir)
 
-    const questionReorganizedRootDir = path.join(dataDir, dataset.database)
-    ensureAndClearDir(questionReorganizedRootDir)
-
-    for (const question of questions) {
+    for await (const question of findAllQuestions()) {
       if (options.verbose) {
-        console.log(`Converting ${question.id} file…`)
+        console.log(`Converting ${question.reference} file…`)
       }
 
-      insertQuestionReferences(question, allData, {})
-
       const legislature = question.legislature ? question.legislature : 0
-      const questionReorganizedDir = path.join(questionReorganizedRootDir, String(legislature))
+      const questionReorganizedDir = path.join(questionsReorganizedRootDir, String(legislature))
       fs.ensureDirSync(questionReorganizedDir)
-      let questionFileName = `${question.reference}.json`
-      fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, {spaces: 2})
+      const questionFileName = `${question.reference}.json`
+      fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 })
     }
   }
 
@@ -150,27 +135,103 @@ async function convertData() {
       console.log(`Converting database ${dataset.database} data into files…`)
     }
 
-    const sens = await getAllSens()
-    const aggregator = new Aggregator(new Set(allFollows))
-    for (const sen of sens) {
-      aggregator.addSen(sen)
+    const sensReorganizedRootDir = path.join(dataDir, dataset.database)
+    const senateursReorganizedDir = path.join(sensReorganizedRootDir, SENS_SENATEURS_FOLDER)
+    const circonscriptionsReorganizedDir = path.join(sensReorganizedRootDir, SENS_CIRCONSCRIPTIONS_FOLDER)
+    const organismesReorganizedDir = path.join(sensReorganizedRootDir, SENS_ORGANISMES_FOLDER)
+    ensureAndClearDir(sensReorganizedRootDir)
+    ensureAndClearDir(senateursReorganizedDir)
+    ensureAndClearDir(circonscriptionsReorganizedDir)
+    ensureAndClearDir(organismesReorganizedDir)
+
+    for await (const sen of findAllSens()) {
+      if (options.verbose) {
+        console.log(`Converting ${sen.matricule} file…`)
+      }
+
+      const senFileName = `${sen.matricule}.json`
+      fs.writeJSONSync(path.join(senateursReorganizedDir, senFileName), sen, { spaces: 2 })
     }
-    await aggregator.getAll()
-    const allData = aggregator.toJson()
 
-    const senReorganizedRootDir = path.join(dataDir, dataset.database)
-    ensureAndClearDir(senReorganizedRootDir)
+    for await (const circonscription of findAllCirconscriptions()) {
+      if (options.verbose) {
+        console.log(`Converting ${circonscription.identifiant} file…`)
+      }
+
+      const circonscriptionFileName = `${circonscription.identifiant}.json`
+      fs.writeJSONSync(
+        path.join(circonscriptionsReorganizedDir, circonscriptionFileName), circonscription, { spaces: 2 }
+      )
+    }
 
-    for (const sen of sens) {
+    for await (const organisme of findAllOrganismes()) {
       if (options.verbose) {
-        console.log(`Converting ${sen.senmat} file…`)
+        console.log(`Converting ${organisme.code} file…`)
       }
 
-      insertSenReferences(sen, allData, {})
+      const organismeFileName = `${organisme.code}.json`
+      fs.writeJSONSync(path.join(organismesReorganizedDir, organismeFileName), organisme, { spaces: 2 })
+    }
+  }
+
+  if (!options.silent) {
+    console.timeEnd("data transformation time")
+  }
+}
+
+async function convertTexteUrls (dataDir: string) {
+  const textesDir = path.join(dataDir, TEXTE_FOLDER)
+  fs.ensureDirSync(textesDir)
+
+  const originalTextesDir = path.join(textesDir, TEXTE_ORIGINAL_FOLDER)
+
+  for await (const texte of findSenatTexteUrls(options.sessions)) {
+    const texteName = path.parse(texte.url).name
+    const texteDir = path.join(originalTextesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
+    fs.ensureDirSync(texteDir)
+
+    const metadata: TexteMetadata = {
+      name: texteName,
+      session: texte.session,
+      url_expose_des_motifs: texte.hasExposeDesMotifs ?
+        new URL(`${texteName}-expose.html`, SENAT_EXPOSE_DES_MOTIFS_BASE_URL) : undefined,
+      url_xml: new URL(`${texteName}.akn.xml`, SENAT_TEXTE_XML_BASE_URL),
+      url_html: new URL(`${texteName}.html`, SENAT_TEXTE_BASE_URL),
+      url_pdf: new URL(`${texteName}.pdf`, SENAT_TEXTE_BASE_URL),
+    }
+    fs.writeJSONSync(path.join(texteDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
+  }
+}
 
-      let senFileName = `${sen.senmat}.json`
-      fs.writeJSONSync(path.join(senReorganizedRootDir, senFileName), sen, {spaces: 2})
+async function convertRapportUrls (dataDir: string) {
+  const rapportsDir = path.join(dataDir, RAPPORT_FOLDER)
+  fs.ensureDirSync(rapportsDir)
+
+  for await (const rapport of findSenatRapportUrls(options.sessions)) {
+    const parsedRapportUrl = path.parse(rapport.url)
+    const rapportName = parsedRapportUrl.name
+    const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName)
+    fs.ensureDirSync(rapportDir)
+
+    const rapportHtmlUrlBase = `${rapportName}_mono.html`
+    const rapportHtmlUrl = path.format({
+      dir: parsedRapportUrl.dir,
+      base: rapportHtmlUrlBase,
+    })
+
+    const rapportPdfUrlBase = `${rapportName}1.pdf`
+    const rapportPdfUrl = path.format({
+      dir: parsedRapportUrl.dir,
+      base: rapportPdfUrlBase,
+    })
+
+    const metadata: RapportMetadata = {
+      name: rapportName,
+      session: rapport.session,
+      url_html: new URL(rapportHtmlUrl, SENAT_RAPPORT_BASE_URL),
+      url_pdf: new URL(rapportPdfUrl, SENAT_RAPPORT_BASE_URL),
     }
+    fs.writeJSONSync(path.join(rapportDir, DOCUMENT_METADATA_FILE), metadata, { spaces: 2 })
   }
 }
 

package/lib/scripts/datautil.d.ts

@@ -0,0 +1,5 @@
+export declare const SIGNET_STRUCTURE_REGEXP: RegExp;
+export declare const AKN_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const AKN_WORKFLOW_IDENTIFICATION_STRUCTURE_REGEXP: RegExp;
+export declare const UNDEFINED_SESSION = "0";
+export declare function formatToFourDigitSession(session: string): string;