@tricoteuses/senat 2.20.9 → 2.20.12

package/README.md

@@ -56,10 +56,10 @@ npm run data:parse_textes_lois ../senat-data
 npm run data:retrieve_agenda ../senat-data -- --fromSession 2022 [--parseAgenda]
 
 # Retrieval (& parsing) of comptes-rendus de séance from Sénat's data
-npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats]
+npm run data:retrieve_cr_seance ../senat-data -- [--parseDebats] [--keepDir]
 
 # Retrieval (& parsing) of comptes-rendus de commissions from Sénat's website
-npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats]
+npm run data:retrieve_cr_commission ../senat-data -- [--parseDebats] [--keepDir]
 
 # Retrieval of sénateurs' pictures from Sénat's website
 npm run data:retrieve_senateurs_photos ../senat-data

package/lib/model/ameli.js

@@ -123,7 +123,7 @@ const findAllAmendementsQuery = dbSenat
     "ameli.avigvt.lib as avis_gouvernement",
     eb.fn.coalesce("ameli.sor.lib", "ameli.irr.libirr").as("sort"),
     "ameli.amd.rev as revision",
-    concat(val("https://www.senat.fr/amendements/"), ref("ameli.ses.ann"), val("-"), sql `(ameli.ses.ann + 1)`, val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.numabs"), val(".html")).as("url"),
+    concat(val("https://www.senat.fr/amendements/"), ref("ameli.ses.ann"), val("-"), sql `(ameli.ses.ann + 1)`, val("/"), ref("ameli.txt_ameli.numabs"), val("/Amdt_"), ref("ameli.amd.num"), val(".html")).as("url"),
     "ameli.grppol_ameli.lilcou as au_nom_de_groupe_politique",
     "ameli.com_ameli.lil as au_nom_de_commission",
     eb.case().when("ameli.cab.entid", "is not", null).then(true).else(false).end().as("auteur_est_gouvernement"),

package/lib/model/dosleg.js

@@ -34,10 +34,12 @@ function rapports(lectureAssembleeId) {
         .withSchema("dosleg")
         .selectFrom("rap")
         .leftJoin("lecassrap", "lecassrap.rapcod", "rap.rapcod")
+        .leftJoin("raporg", "raporg.rapcod", "rap.rapcod")
         .leftJoin("denrap", "denrap.coddenrap", "rap.coddenrap")
         .where("lecassrap.lecassidt", "=", lectureAssembleeId)
         .select(({ eb, ref, val }) => [
         "rap.rapnum as numero",
+        "raporg.orgcod as code_organisme",
         eb
             .case()
             .when("rap.typurl", "=", "I")
@@ -84,6 +86,7 @@ function textes(lectureAssembleeId) {
         .where("texte.lecassidt", "=", lectureAssembleeId)
         .select(({ eb, ref, val }) => [
         "texte.texnum as numero",
+        "texte.orgcod as code_organisme",
         eb
             .case()
             .when("texte.typurl", "=", "I")

package/lib/scripts/retrieve_cr_commission.js

@@ -1,4 +1,4 @@
-import fs from "fs-extra";
+import fs, { ensureDir } from "fs-extra";
 import assert from "assert";
 import path from "path";
 import * as cheerio from "cheerio";
@@ -187,7 +187,12 @@ async function retrieveCommissionCRs(options = {}) {
     const politenessMs = Number(options["politenessMs"] ?? 150);
     const commissionsRootDir = path.join(dataDir, COMMISSION_FOLDER);
     const originalRoot = path.join(commissionsRootDir, DATA_ORIGINAL_FOLDER);
-    ensureAndClearDir(originalRoot);
+    if (!options["keepDir"]) {
+        ensureAndClearDir(originalRoot);
+    }
+    else {
+        ensureDir(originalRoot);
+    }
     const discovered = await discoverCommissionWeeklyPages(fromSession);
     console.log(`[COM-CR][discover] ${discovered.length} links (>= session ${fromSession})`);
     const jobs = discovered.map(({ url, yyyymmdd, commissionKey }) => {
@@ -238,7 +243,9 @@ async function retrieveCommissionCRs(options = {}) {
     const sessions = getSessionsFromStart(options["fromSession"]);
     const comRoot = path.join(dataDir, COMMISSION_FOLDER);
     const transformedRoot = path.join(comRoot, DATA_TRANSFORMED_FOLDER);
-    if (options["parseDebats"])
+    if (options["keepDir"])
+        ensureDir(transformedRoot);
+    else
         ensureAndClearDir(transformedRoot);
     for (const session of sessions) {
         const originalSessionDir = path.join(originalRoot, String(session));

package/lib/scripts/retrieve_open_data.js

@@ -116,13 +116,19 @@ async function copyToSenat(dataset, dataDir, options) {
                 execSync(`${options["sudo"] ? `sudo -u ${options["sudo"]} ` : ""}psql --quiet -d senat -f ${schemaDumpFile}`, {
                     env: process.env,
                     encoding: "utf-8",
-                    stdio: ["ignore", "ignore", "pipe"],
+                    stdio: ["ignore", "pipe", "pipe"],
                 });
             }
-            finally {
-                try {
+            catch (error) {
+                if (!options["silent"]) {
+                    console.error(`Failed to import ${dataset.database} schema:`);
+                    if (error.stderr) {
+                        console.error(error.stderr);
+                    }
+                    if (error.stdout) {
+                        console.error(error.stdout);
+                    }
                 }
-                catch { }
             }
             resolve();
         });

package/lib/scripts/shared/cli_helpers.d.ts

@@ -35,6 +35,11 @@ export declare const onlyRecentOption: {
     name: string;
     type: NumberConstructor;
 };
+export declare const keepDirOption: {
+    help: string;
+    name: string;
+    type: BooleanConstructor;
+};
 export declare const commonOptions: ({
     alias: string;
     defaultValue: string[];
@@ -48,12 +53,11 @@ export declare const commonOptions: ({
     name: string;
     type: StringConstructor;
 } | {
-    alias: string;
     help: string;
     name: string;
-    type: BooleanConstructor;
+    type: NumberConstructor;
 } | {
     help: string;
     name: string;
-    type: NumberConstructor;
+    type: BooleanConstructor;
 })[];

package/lib/scripts/shared/cli_helpers.js

@@ -35,6 +35,11 @@ export const onlyRecentOption = {
     name: "only-recent",
     type: Number,
 };
+export const keepDirOption = {
+    help: "keep directories when cleaning data",
+    name: "keepDir",
+    type: Boolean,
+};
 export const commonOptions = [
     categoriesOption,
     dataDirDefaultOption,
@@ -42,4 +47,5 @@ export const commonOptions = [
     silentOption,
     verboseOption,
     onlyRecentOption,
+    keepDirOption,
 ];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tricoteuses/senat",
-  "version": "2.20.9",
+  "version": "2.20.12",
   "description": "Handle French Sénat's open data",
   "keywords": [
     "France",