@tricoteuses/senat 1.3.1 → 1.3.4

package/lib/scripts/retrieve_textes.ts

@@ -1,143 +0,0 @@
-import assert from "assert"
-import commandLineArgs from "command-line-args"
-import fs from "fs-extra"
-import path from "path"
-
-import { findSenatRapportUrls, findSenatTexteUrls } from "../model/dosleg"
-import { UNDEFINED_SESSION } from "./datautil"
-import { commonOptions } from "./shared/cli_helpers"
-import { ensureAndClearDir, fetchWithRetry } from "./shared/util"
-
-const optionsDefinitions = [
-  ...commonOptions,
-  {
-    help: "sessions of texts to retrieve; leave empty for all",
-    multiple: true,
-    name: "sessions",
-    type: String,
-  },
-  {
-    help: "parse and convert documents into JSON (textes only for now)",
-    name: "parseDocuments",
-    type: Boolean,
-  },
-]
-const options = commandLineArgs(optionsDefinitions)
-
-const SENAT_TEXTE_BASE_URL = "https://www.senat.fr/akomantoso/"
-const SENAT_RAPPORT_BASE_URL = "https://www.senat.fr/rap/"
-
-async function retrieveDocument (documentUrl: string) {
-  if (!options.silent) {
-    console.log(`Retrieving document ${documentUrl}…`)
-  }
-
-  try {
-    const response = await fetchWithRetry(documentUrl)
-    if (!response.ok) {
-      if (response.status === 404) {
-        console.warn(`Texte ${documentUrl} not found`)
-      } else {
-        console.error(`An error occurred while retrieving texte ${documentUrl}: ${response.status}`)
-      }
-      return null
-    }
-    return await response.text()
-  } catch (error: any) {
-    console.error(error.message)
-    return null
-  }
-}
-
-async function retrieveTextes (dataDir: string) {
-  const textesDir = path.join(dataDir, "leg")
-  fs.ensureDirSync(textesDir)
-
-  let retrievedTextesCount = 0
-  const texteUrlsNotFoundOrError = []
-
-  for await (const texte of findSenatTexteUrls(options.sessions)) {
-    const texteName = path.parse(texte.url).name
-    const texteXmlUrl = `${texteName}.akn.xml`
-    const texteXmlAbsoluteUrl = new URL(texteXmlUrl, SENAT_TEXTE_BASE_URL).toString()
-
-    const texteXml = await retrieveDocument(texteXmlAbsoluteUrl)
-
-    if (texteXml) {
-      if (options.parseDocuments) {
-        //const parsedTexte = await parseTexte(texteXml)
-      }
-      
-      const texteDir = path.join(textesDir, `${texte.session ?? UNDEFINED_SESSION}`, texteName)
-      ensureAndClearDir(texteDir)
-      fs.writeFileSync(path.join(texteDir, texteXmlUrl), texteXml)
-      retrievedTextesCount++
-    } else {
-      texteUrlsNotFoundOrError.push(texteXmlAbsoluteUrl)
-    }
-  }
-
-  if (options.verbose) {
-    console.log(`${retrievedTextesCount} textes retrieved`)
-    console.log(
-      `${texteUrlsNotFoundOrError.length} textes failed with URLs ${texteUrlsNotFoundOrError.join(", ")}`
-    )
-  }
-  // TODO retrieve other formats (html/pdf) ?
-}
-
-async function retrieveRapports (dataDir: string) {
-  const rapportsDir = path.join(dataDir, "rap")
-  fs.ensureDirSync(rapportsDir)
-
-  let retrievedRapportsCount = 0
-  const rapportUrlsNotFoundOrError = []
-
-  for await (const rapport of findSenatRapportUrls(options.sessions)) {
-    let rapportUrl = rapport.url
-    const parsedRapportUrl = path.parse(rapport.url)
-    let rapportUrlBase = parsedRapportUrl.base
-    const rapportName = parsedRapportUrl.name
-    if (parsedRapportUrl.ext === ".html") {
-      rapportUrlBase = `${rapportName}_mono.html`
-      rapportUrl = path.format({
-        dir: parsedRapportUrl.dir,
-        base: rapportUrlBase,
-      })
-    }
-    const rapportAbsoluteUrl = new URL(rapportUrl, SENAT_RAPPORT_BASE_URL).toString()
-
-    const rapportContent = await retrieveDocument(rapportAbsoluteUrl)
-
-    if (rapportContent) {
-      const rapportDir = path.join(rapportsDir, `${rapport.session ?? UNDEFINED_SESSION}`, rapportName)
-      ensureAndClearDir(rapportDir)
-      fs.writeFileSync(path.join(rapportDir, rapportUrlBase), rapportContent)
-      retrievedRapportsCount++
-    } else {
-      rapportUrlsNotFoundOrError.push(rapportAbsoluteUrl)
-    }
-  }
-
-  if (options.verbose) {
-    console.log(`${retrievedRapportsCount} rapports retrieved`)
-    console.log(
-      `${rapportUrlsNotFoundOrError.length} rapports failed with URLs ${rapportUrlsNotFoundOrError.join(", ")}`
-    )
-  }
-}
-
-async function main() {
-  const dataDir = options.dataDir
-  assert(dataDir, "Missing argument: data directory")
-
-  await retrieveTextes(dataDir)
-  await retrieveRapports(dataDir)
-}
-
-main()
-  .then(() => process.exit(0))
-  .catch((error) => {
-    console.log(error)
-    process.exit(1)
-  })

package/lib/scripts/shared/cli_helpers.ts

@@ -1,36 +0,0 @@
-export const categoriesOption = {
-  alias: "k",
-  defaultValue: ["All"],
-  help: "categories of datasets to reorganize",
-  multiple: true,
-  name: "categories",
-  type: String,
-}
-
-export const dataDirDefaultOption = {
-  defaultOption: true,
-  help: "directory containing Sénat open data files",
-  name: "dataDir",
-  type: String,
-}
-
-export const silentOption = {
-  alias: "s",
-  help: "don't log anything",
-  name: "silent",
-  type: Boolean,
-}
-
-export const verboseOption = {
-  alias: "v",
-  help: "verbose logs",
-  name: "verbose",
-  type: Boolean,
-}
-
-export const commonOptions = [
-  categoriesOption,
-  dataDirDefaultOption,
-  silentOption,
-  verboseOption,
-]

package/lib/scripts/shared/util.ts

@@ -1,33 +0,0 @@
-import fs from "fs-extra"
-
-export function isOptionEmptyOrHasValue (option: string, value: string) {
-  return !option || option.length === 0 || option.includes(value)
-}
-
-export function ensureAndClearDir (path: string) {
-  if (!fs.existsSync(path)) {
-    fs.mkdirSync(path, { recursive: true })
-  } else {
-    fs.emptyDirSync(path)
-  }
-}
-
-export async function fetchWithRetry (
-  url: string,
-  retries: number = 3,
-  backoff: number = 300,
-): Promise<Response> {
-  for (let attempt = 0; attempt < retries; attempt++) {
-    try {
-      return await fetch(url)
-    } catch (error) {
-      if (attempt === retries - 1) {
-        throw error
-      }
-      console.warn(`Fetch attempt ${attempt + 1} for ${url} failed. Retrying in ${backoff}ms…`)
-      await new Promise((resolve) => setTimeout(resolve, backoff))
-      backoff *= 2
-    }
-  }
-  throw new Error(`Failed to fetch ${url} after ${retries} attempts`)
-}

package/lib/src/aggregates.d.ts

@@ -1,52 +0,0 @@
-import { Ses, Sub, TxtAmeli } from "./types/ameli";
-import { Debat } from "./types/debats";
-import { Ass, Aud, Auteur, DateSeance, DecCoc, DenRap, DocAtt, Ecr, EtaLoi, LecAss, LecAssRap, Lecture, Loi, Org, OriTxt, Qua, Rap, Scr, Texte, TypAtt, TypLec, TypLoi, TypTxt, TypUrl } from "./types/dosleg";
-import { Question } from "./types/questions";
-export declare const allFollows: string[];
-export declare class Aggregator {
-    follow: Set<string>;
-    objectByIdByTableName: {
-        [name: string]: {
-            [id: string]: object;
-        };
-    };
-    requestedIdsByTableName: {
-        [name: string]: Set<string>;
-    };
-    visitedIdsByTableName: {
-        [name: string]: Set<string>;
-    };
-    constructor(follow: Set<string>);
-    addAss(ass: Ass): void;
-    addAud(aud: Aud): void;
-    addAuteur(auteur: Auteur): void;
-    addDateSeance(dateSeance: DateSeance): void;
-    addDebat(debat: Debat): void;
-    addDeccoc(deccoc: DecCoc): void;
-    addDenrap(denrap: DenRap): void;
-    addDocatt(docatt: DocAtt): void;
-    addEcr(ecr: Ecr): void;
-    addEtaloi(etaloi: EtaLoi): void;
-    addLecass(lecass: LecAss): void;
-    addLecassrap(lecassrap: LecAssRap): void;
-    addLecture(lecture: Lecture): void;
-    addLoi(loi: Loi): void;
-    addOrg(org: Org): void;
-    addOritxt(oritxt: OriTxt): void;
-    addQua(qua: Qua): void;
-    addQuestion(question: Question): void;
-    addRap(rap: Rap): void;
-    addScr(scr: Scr): void;
-    addSes(ses: Ses): void;
-    addSub(sub: Sub): void;
-    addTexte(texte: Texte): void;
-    addTxtAmeli(txtAmeli: TxtAmeli): void;
-    addTypatt(typatt: TypAtt): void;
-    addTyplec(typlec: TypLec): void;
-    addTyploi(typloi: TypLoi): void;
-    addTyptxt(typtxt: TypTxt): void;
-    addTypurl(typurl: TypUrl): void;
-    getAll(): Promise<void>;
-    request(tableName: string, id: string): void;
-    toJson(): any;
-}