@tricoteuses/senat 1.1.2 → 1.3.2

package/lib/scripts/retrieve_textes.mjs

@@ -1,77 +0,0 @@
-import assert from "assert";
-import commandLineArgs from "command-line-args";
-import fetch from "cross-fetch";
-import fs from "fs-extra";
-import path from "path";
-import { texteFieldsToParseInt, texteFieldsToTrim } from "../types/dosleg";
-import { checkDatabase } from "../databases";
-import { parseIntFields, trimFieldsRight } from "../fields";
-const optionsDefinitions = [
-    {
-        alias: "s",
-        help: "don't log anything",
-        name: "silent",
-        type: Boolean,
-    },
-    {
-        alias: "v",
-        help: "verbose logs",
-        name: "verbose",
-        type: Boolean,
-    },
-    {
-        defaultOption: true,
-        help: "directory containing Sénat open data files",
-        name: "dataDir",
-        type: String,
-    },
-];
-const options = commandLineArgs(optionsDefinitions);
-async function main() {
-    const dataDir = options.dataDir;
-    assert(dataDir, "Missing argument: data directory");
-    const db = await checkDatabase("dosleg");
-    const legDir = path.join(dataDir, "leg");
-    fs.ensureDirSync(legDir);
-    const textes = (await db.any(`
-      SELECT *
-      FROM texte
-    `)).map((texte) => parseIntFields(texteFieldsToParseInt, trimFieldsRight(texteFieldsToTrim, texte)));
-    for (const texte of textes) {
-        if (texte.texurl === null) {
-            continue;
-        }
-        if (texte.typurl !== "I") {
-            // This is not an "I"nternal URL.
-            continue;
-        }
-        if (texte.texurl.endsWith(".pdf")) {
-            continue;
-        }
-        const url = new URL(texte.texurl, "http://www.senat.fr/leg/").toString();
-        if (!options.silent) {
-            console.log(`Retrieving page ${url}…`);
-        }
-        const response = await fetch(url);
-        const page = await response.text();
-        if (!response.ok) {
-            if (response.status === 404) {
-                console.log(`Page ${url} not found`);
-            }
-            else {
-                console.error(`An error occurred while retrieving page ${url}: ${response.status} ${response.statusText}\n${page}`);
-            }
-            continue;
-        }
-        const pathParsed = path.parse(path.join(legDir, ...texte.texurl.split("/").filter(Boolean)));
-        const pageDir = path.join(pathParsed.dir, pathParsed.name);
-        fs.ensureDirSync(pageDir);
-        fs.writeFileSync(path.join(pageDir, "index" + pathParsed.ext), page);
-    }
-}
-main()
-    .then(() => process.exit(0))
-    .catch((error) => {
-    console.log(error);
-    process.exit(1);
-});

package/lib/scripts/retrieve_textes.ts

@@ -1,95 +0,0 @@
-import assert from "assert"
-import commandLineArgs from "command-line-args"
-import fetch from "cross-fetch"
-import fs from "fs-extra"
-import path from "path"
-
-import { Texte, texteFieldsToParseInt, texteFieldsToTrim } from "../types/dosleg"
-import { checkDatabase } from "../databases"
-import { parseIntFields, trimFieldsRight } from "../fields"
-
-const optionsDefinitions = [
-  {
-    alias: "s",
-    help: "don't log anything",
-    name: "silent",
-    type: Boolean,
-  },
-  {
-    alias: "v",
-    help: "verbose logs",
-    name: "verbose",
-    type: Boolean,
-  },
-  {
-    defaultOption: true,
-    help: "directory containing Sénat open data files",
-    name: "dataDir",
-    type: String,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-
-async function main() {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  const db = await checkDatabase("dosleg")
-
-  const legDir = path.join(dataDir, "leg")
-  fs.ensureDirSync(legDir)
-  const textes: Texte[] = (
-    await db.any(
-      `
-      SELECT *
-      FROM texte
-    `,
-    )
-  ).map((texte: Texte) =>
-    parseIntFields(
-      texteFieldsToParseInt,
-      trimFieldsRight(texteFieldsToTrim, texte),
-    ),
-  )
-  for (const texte of textes) {
-    if (texte.texurl === null) {
-      continue
-    }
-    if (texte.typurl !== "I") {
-      // This is not an "I"nternal URL.
-      continue
-    }
-    if (texte.texurl.endsWith(".pdf")) {
-      continue
-    }
-    const url = new URL(texte.texurl, "http://www.senat.fr/leg/").toString()
-    if (!options.silent) {
-      console.log(`Retrieving page ${url}…`)
-    }
-    const response = await fetch(url)
-    const page = await response.text()
-    if (!response.ok) {
-      if (response.status === 404) {
-        console.log(`Page ${url} not found`)
-      } else {
-        console.error(
-          `An error occurred while retrieving page ${url}: ${response.status} ${response.statusText}\n${page}`,
-        )
-      }
-      continue
-    }
-    const pathParsed = path.parse(
-      path.join(legDir, ...texte.texurl.split("/").filter(Boolean)),
-    )
-    const pageDir = path.join(pathParsed.dir, pathParsed.name)
-    fs.ensureDirSync(pageDir)
-    fs.writeFileSync(path.join(pageDir, "index" + pathParsed.ext), page)
-  }
-}
-
-main()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/strings.mjs

@@ -1,18 +0,0 @@
-import originalSlugify from "slug";
-const slugifyCharmap = {
-    ...originalSlugify.defaults.charmap,
-    "'": " ",
-    "@": " ",
-    ".": " ",
-    "-": "_",
-};
-export function slugify(string, replacement) {
-    const options = {
-        charmap: slugifyCharmap,
-        mode: "rfc3986",
-    };
-    if (replacement) {
-        options.replacement = replacement;
-    }
-    return originalSlugify(string, options);
-}

package/lib/strings.ts

@@ -1,26 +0,0 @@
-import originalSlugify from "slug"
-
-const slugifyCharmap = {
-  ...originalSlugify.defaults.charmap,
-  "'": " ",
-  "@": " ",
-  ".": " ",
-  "-": "_",
-}
-
-interface Options {
-  charmap: { [c: string]: string }
-  mode: string
-  replacement?: string
-}
-
-export function slugify(string: string, replacement?: string): string {
-  const options: Options = {
-    charmap: slugifyCharmap,
-    mode: "rfc3986",
-  }
-  if (replacement) {
-    options.replacement = replacement
-  }
-  return originalSlugify(string, options)
-}

package/lib/types/ameli.mjs

@@ -1,13 +0,0 @@
-export const sesFieldsToParseInt = ["ann"];
-export const sesFieldsToTrim = ["lil"];
-export const subFieldsToParseInt = ["pos", "posder", "prires"];
-export const subFieldsToTrim = ["lic", "lib", "sig", "style"];
-export const txtAmeliFieldsToTrim = [
-    "num",
-    "int",
-    "inl",
-    "libdelim",
-    "libcplnat",
-    "doslegsignet",
-    "ordsnddelib",
-];

package/lib/types/ameli.ts

@@ -1,21 +0,0 @@
-import { ses as Ses, sub as Sub, subFields, txt_ameli } from "../raw_types/ameli"
-
-export { Ses, Sub }
-
-export interface TxtAmeli extends txt_ameli {
-  subids?: subFields.id[]
-}
-
-export const sesFieldsToParseInt = ["ann"]
-export const sesFieldsToTrim = ["lil"]
-export const subFieldsToParseInt = ["pos", "posder", "prires"]
-export const subFieldsToTrim = ["lic", "lib", "sig", "style"]
-export const txtAmeliFieldsToTrim = [
-  "num",
-  "int",
-  "inl",
-  "libdelim",
-  "libcplnat",
-  "doslegsignet",
-  "ordsnddelib",
-]

package/lib/types/debats.mjs

@@ -1,2 +0,0 @@
-export const debatsFieldsToTrim = ["deburl", "libspec"];
-export const lecassdebFieldsToTrim = ["lecassidt"];

package/lib/types/debats.ts

@@ -1,6 +0,0 @@
-import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types/debats"
-
-export { Debat, LecAssDeb }
-
-export const debatsFieldsToTrim = ["deburl", "libspec"]
-export const lecassdebFieldsToTrim = ["lecassidt"]

package/lib/types/dosleg.mjs

@@ -1,151 +0,0 @@
-export const assFieldsToTrim = ["libass"];
-export const audFieldsToTrim = ["lecassidt", "audtit", "audurl", "orgcod"];
-export const auteurFieldsToTrim = [
-    "autcod",
-    "quacod",
-    "typautcod",
-    "nomuse",
-    "prenom",
-    "nomtec",
-    "autmat",
-    "autfct",
-];
-export const dateSeanceFieldsToTrim = ["lecidt", "statut"];
-export const deccocFieldsToTrim = ["deccoccod", "deccoclib"];
-export const denrapFieldsToTrim = [
-    "coddenrap",
-    "typraprap",
-    "libdenrap",
-    "denrapmin",
-    "denraptit",
-    "denrapstymin",
-    "solnatrapcod",
-];
-export const docattFieldsToParseInt = ["docattcle", "rapcod"];
-export const docattFieldsToTrim = ["docatturl"];
-export const ecrFieldsToTrim = ["autcod", "ecrqua"];
-export const etaloiFieldsToTrim = ["etaloilib"];
-export const lecassFieldsToTrim = [
-    "lecassidt",
-    "lecidt",
-    "ptlurl",
-    "ptlnumcpl",
-    "ptlnot",
-    "ptlurl2",
-    "ptlnot2",
-    "ptlurl3",
-    "ptlnot3",
-    "ptlnumcpl2",
-    "ptlnumcpl3",
-    "lecassame",
-    "orgcod",
-    "loiintmod",
-    "reucom",
-    "debatsurl",
-    "libppr",
-    "ptlurlcom",
-    "aliasppr",
-    "lecassamecom",
-];
-export const lecassrapFieldsToTrim = ["lecassidt"];
-export const lectureFieldsToTrim = ["lecidt", "loicod", "typleccod", "leccom"];
-export const loiFieldsToTrim = [
-    "loicod",
-    "typloicod",
-    "deccoccod",
-    "numero",
-    "loient",
-    "motclef",
-    "loitit",
-    "loiint",
-    "url_jo",
-    "loinumjo",
-    "loititjo",
-    "url_jo2",
-    "loinumjo2",
-    "deccocurl",
-    "num_decision",
-    "loicodmai",
-    "loinoudelibcod",
-    "motionloiorigcod",
-    "url_ordonnance",
-    "saisine_par",
-    "loinumjo3",
-    "url_jo3",
-    "url_an",
-    "url_presart",
-    "signetalt",
-    "orgcod",
-    "doscocurl",
-    "loiintori",
-];
-export const orgFieldsToTrim = [
-    "orgcod",
-    "typorgcod",
-    "orgnom",
-    "orgliblon",
-    "orglibaff",
-    "orgurl",
-    "orglibcou",
-    "org_de",
-    "urltra",
-    "inttra",
-    "orgnomcouv",
-    "senorgcod",
-    "html_color",
-];
-export const oritxtFieldsToTrim = ["oritxtcod", "oritxtlib", "oritxtlibfem"];
-export const quaFieldsToTrim = ["quacod", "qualic", "quaabr", "quaabrplu"];
-export const rapFieldsToParseInt = ["rapcod", "sesann", "rapnum", "rapnuman"];
-export const rapFieldsToTrim = [
-    "coddenrap",
-    "blecod",
-    "raptitcou",
-    "raptil",
-    "rapurl",
-    "url2",
-    "url3",
-    "url4",
-    "url2txt",
-    "url3txt",
-    "url4txt",
-    "prix",
-    "numerobis",
-    "rapsoustit",
-    "rapres",
-    "forpubcod",
-];
-export const raporgFieldsToTrim = ["orgcod"];
-export const scrFieldsToTrim = ["scrint", "soslib"];
-export const texteFieldsToParseInt = ["texcod", "sesann", "texnum"];
-export const texteFieldsToTrim = [
-    "oritxtcod",
-    "typtxtcod",
-    "lecassidt",
-    "orgcod",
-    "texurl",
-    "url2",
-    "url3",
-    "url4",
-    "url2txt",
-    "url3txt",
-    "url4txt",
-    "prix",
-    "numerobis",
-    "reserve_comspe",
-];
-export const typattFieldsToTrim = ["typattlib"];
-export const typlecFieldsToTrim = ["typleccod", "typleclib"];
-export const typloiFieldsToTrim = [
-    "typloicod",
-    "typloilib",
-    "groupe",
-    "typloiden",
-    "typloigen",
-    "typloitit",
-    "typloidenplu",
-    "typloide",
-    "typloiabr",
-];
-export const typtxtFieldsToTrim = ["typtxtcod", "typtxtlib"];
-export const typurlFieldsToTrim = ["libtypurl"];

package/lib/types/dosleg.ts

@@ -1,284 +0,0 @@
-import { txt_ameliFields } from "../raw_types/ameli"
-import {
-  ass as Ass,
-  aud,
-  audFields,
-  auteur,
-  date_seance,
-  date_seanceFields as dateSeanceFields,
-  deccoc as DecCoc,
-  denrap as DenRap,
-  docatt,
-  docattFields,
-  ecr,
-  ecrFields,
-  etaloi as EtaLoi,
-  lecass,
-  lecassFields,
-  lecassrap,
-  lecture,
-  lectureFields,
-  loi,
-  org as Org,
-  oritxt as OriTxt,
-  qua as Qua,
-  rap,
-  raporg as RapOrg,
-  raporgFields,
-  scr as Scr,
-  texte,
-  texteFields,
-  typatt as TypAtt,
-  typlec as TypLec,
-  typloi as TypLoi,
-  typtxt as TypTxt,
-  typurl as TypUrl,
-  typurlFields,
-} from "../raw_types/dosleg"
-import { Debat } from "./debats"
-import { TxtAmeli } from "./ameli"
-
-export {
-  Ass,
-  DecCoc,
-  DenRap,
-  EtaLoi,
-  Org,
-  OriTxt,
-  Qua,
-  RapOrg,
-  Scr,
-  TypAtt,
-  TypLec,
-  TypLoi,
-  TypTxt,
-  TypUrl,
-}
-
-export interface Aud extends aud {
-  org?: Org
-}
-
-export interface Auteur extends auteur {
-  qua?: Qua
-}
-
-export interface DateSeance extends date_seance {
-  debat?: Debat
-  scrids?: string[]
-  scrs?: Scr[]
-}
-
-export interface DocAtt extends docatt {
-  rap?: Rap
-  typatt?: TypAtt
-}
-
-export interface Ecr extends ecr {
-  aut?: Auteur
-}
-
-export interface LecAss extends lecass {
-  ass?: Ass
-  auds?: Aud[]
-  audcles?: audFields.audcle[]
-  datesSeances?: DateSeance[]
-  datesSeancesCodes?: dateSeanceFields.code[]
-  debatdatseas: Date[]
-  lecassraps?: LecAssRap[]
-  lecassrapids?: string[]
-  org?: Org
-  texcods: texteFields.texcod[]
-  textes?: Texte[]
-}
-
-export interface LecAssRap extends lecassrap {
-  rap?: Rap
-}
-
-export interface Lecture extends lecture {
-  typlec?: TypLec
-  lecassidts?: lecassFields.lecassidt[]
-  lecasss?: LecAss[]
-}
-
-export interface Loi extends loi {
-  deccoc?: DecCoc
-  etaloi?: EtaLoi
-  lecidts?: lectureFields.lecidt[]
-  lectures?: Lecture[]
-  typloi?: TypLoi
-}
-
-export interface Rap extends rap {
-  denrap?: DenRap
-  docattcles?: docattFields.docattcle[]
-  docatts?: DocAtt[]
-  ecrnums?: ecrFields.ecrnum[]
-  ecrs?: Ecr[]
-  orgcods?: raporgFields.orgcod[]
-  orgs?: Org[]
-}
-
-export interface Texte extends texte {
-  ecrs?: Ecr[]
-  ecrnums?: ecrFields.ecrnum[]
-  libtypurl?: typurlFields.libtypurl
-  org?: Org
-  oritxt: OriTxt
-  typtxt?: TypTxt
-  txtAmeli?: TxtAmeli
-  txtAmeliId: txt_ameliFields.id
-}
-
-export const assFieldsToTrim = ["libass"]
-export const audFieldsToTrim = ["lecassidt", "audtit", "audurl", "orgcod"]
-export const auteurFieldsToTrim = [
-  "autcod",
-  "quacod",
-  "typautcod",
-  "nomuse",
-  "prenom",
-  "nomtec",
-  "autmat",
-  "autfct",
-]
-export const dateSeanceFieldsToTrim = ["lecidt", "statut"]
-export const deccocFieldsToTrim = ["deccoccod", "deccoclib"]
-export const denrapFieldsToTrim = [
-  "coddenrap",
-  "typraprap",
-  "libdenrap",
-  "denrapmin",
-  "denraptit",
-  "denrapstymin",
-  "solnatrapcod",
-]
-export const docattFieldsToParseInt = ["docattcle", "rapcod"]
-export const docattFieldsToTrim = ["docatturl"]
-export const ecrFieldsToTrim = ["autcod", "ecrqua"]
-export const etaloiFieldsToTrim = ["etaloilib"]
-export const lecassFieldsToTrim = [
-  "lecassidt",
-  "lecidt",
-  "ptlurl",
-  "ptlnumcpl",
-  "ptlnot",
-  "ptlurl2",
-  "ptlnot2",
-  "ptlurl3",
-  "ptlnot3",
-  "ptlnumcpl2",
-  "ptlnumcpl3",
-  "lecassame",
-  "orgcod",
-  "loiintmod",
-  "reucom",
-  "debatsurl",
-  "libppr",
-  "ptlurlcom",
-  "aliasppr",
-  "lecassamecom",
-]
-export const lecassrapFieldsToTrim = ["lecassidt"]
-export const lectureFieldsToTrim = ["lecidt", "loicod", "typleccod", "leccom"]
-export const loiFieldsToTrim = [
-  "loicod",
-  "typloicod",
-  "deccoccod",
-  "numero",
-  "loient",
-  "motclef",
-  "loitit",
-  "loiint",
-  "url_jo",
-  "loinumjo",
-  "loititjo",
-  "url_jo2",
-  "loinumjo2",
-  "deccocurl",
-  "num_decision",
-  "loicodmai",
-  "loinoudelibcod",
-  "motionloiorigcod",
-  "url_ordonnance",
-  "saisine_par",
-  "loinumjo3",
-  "url_jo3",
-  "url_an",
-  "url_presart",
-  "signetalt",
-  "orgcod",
-  "doscocurl",
-  "loiintori",
-]
-export const orgFieldsToTrim = [
-  "orgcod",
-  "typorgcod",
-  "orgnom",
-  "orgliblon",
-  "orglibaff",
-  "orgurl",
-  "orglibcou",
-  "org_de",
-  "urltra",
-  "inttra",
-  "orgnomcouv",
-  "senorgcod",
-  "html_color",
-]
-export const oritxtFieldsToTrim = ["oritxtcod", "oritxtlib", "oritxtlibfem"]
-export const quaFieldsToTrim = ["quacod", "qualic", "quaabr", "quaabrplu"]
-export const rapFieldsToParseInt = ["rapcod", "sesann", "rapnum", "rapnuman"]
-export const rapFieldsToTrim = [
-  "coddenrap",
-  "blecod",
-  "raptitcou",
-  "raptil",
-  "rapurl",
-  "url2",
-  "url3",
-  "url4",
-  "url2txt",
-  "url3txt",
-  "url4txt",
-  "prix",
-  "numerobis",
-  "rapsoustit",
-  "rapres",
-  "forpubcod",
-]
-export const raporgFieldsToTrim = ["orgcod"]
-export const scrFieldsToTrim = ["scrint", "soslib"]
-export const texteFieldsToParseInt = ["texcod", "sesann", "texnum"]
-export const texteFieldsToTrim = [
-  "oritxtcod",
-  "typtxtcod",
-  "lecassidt",
-  "orgcod",
-  "texurl",
-  "url2",
-  "url3",
-  "url4",
-  "url2txt",
-  "url3txt",
-  "url4txt",
-  "prix",
-  "numerobis",
-  "reserve_comspe",
-]
-export const typattFieldsToTrim = ["typattlib"]
-export const typlecFieldsToTrim = ["typleccod", "typleclib"]
-export const typloiFieldsToTrim = [
-  "typloicod",
-  "typloilib",
-  "groupe",
-  "typloiden",
-  "typloigen",
-  "typloitit",
-  "typloidenplu",
-  "typloide",
-  "typloiabr",
-]
-export const typtxtFieldsToTrim = ["typtxtcod", "typtxtlib"]
-export const typurlFieldsToTrim = ["libtypurl"]

package/lib/types/questions.mjs

@@ -1 +0,0 @@
-export {};

package/lib/types/questions.ts

@@ -1,3 +0,0 @@
-import { tam_questions as Question } from "../raw_types/questions"
-
-export { Question }