@tricoteuses/senat 3.2.1 → 3.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/LICENSE.md +32 -32
  2. package/README.md +326 -326
  3. package/lib/src/config.d.ts +43 -0
  4. package/lib/src/config.js +37 -0
  5. package/lib/src/conversion_textes.d.ts +11 -0
  6. package/lib/src/conversion_textes.js +320 -0
  7. package/lib/src/databases_postgres.d.ts +4 -0
  8. package/lib/src/databases_postgres.js +23 -0
  9. package/lib/src/datasets.d.ts +38 -0
  10. package/lib/src/datasets.js +247 -0
  11. package/lib/src/git.d.ts +27 -0
  12. package/lib/src/git.js +251 -0
  13. package/lib/src/index.d.ts +1 -1
  14. package/lib/src/loaders.d.ts +52 -0
  15. package/lib/src/loaders.js +260 -0
  16. package/lib/src/model/agenda.d.ts +6 -0
  17. package/lib/src/model/agenda.js +148 -0
  18. package/lib/src/model/ameli.d.ts +67 -0
  19. package/lib/src/model/ameli.js +150 -0
  20. package/lib/src/model/commission.d.ts +19 -0
  21. package/lib/src/model/commission.js +269 -0
  22. package/lib/src/model/debats.d.ts +39 -0
  23. package/lib/src/model/debats.js +112 -0
  24. package/lib/src/model/documents.d.ts +32 -0
  25. package/lib/src/model/documents.js +182 -0
  26. package/lib/src/model/dosleg.d.ts +144 -0
  27. package/lib/src/model/dosleg.js +468 -0
  28. package/lib/src/model/index.d.ts +7 -0
  29. package/lib/src/model/index.js +7 -0
  30. package/lib/src/model/questions.d.ts +54 -0
  31. package/lib/src/model/questions.js +91 -0
  32. package/lib/src/model/scrutins.d.ts +48 -0
  33. package/lib/src/model/scrutins.js +121 -0
  34. package/lib/src/model/seance.d.ts +3 -0
  35. package/lib/src/model/seance.js +267 -0
  36. package/lib/src/model/sens.d.ts +112 -0
  37. package/lib/src/model/sens.js +385 -0
  38. package/lib/src/model/util.d.ts +1 -0
  39. package/lib/src/model/util.js +15 -0
  40. package/lib/src/parsers/plf/amendement.d.ts +3 -0
  41. package/lib/src/parsers/plf/amendement.js +152 -0
  42. package/lib/src/raw_types/ameli.d.ts +1762 -0
  43. package/lib/src/raw_types/ameli.js +1074 -0
  44. package/lib/src/raw_types/debats.d.ts +380 -0
  45. package/lib/src/raw_types/debats.js +266 -0
  46. package/lib/src/raw_types/dosleg.d.ts +2954 -0
  47. package/lib/src/raw_types/dosleg.js +2005 -0
  48. package/lib/src/raw_types/questions.d.ts +699 -0
  49. package/lib/src/raw_types/questions.js +493 -0
  50. package/lib/src/raw_types/sens.d.ts +7843 -0
  51. package/lib/src/raw_types/sens.js +4691 -0
  52. package/lib/src/raw_types_schemats/ameli.d.ts +541 -0
  53. package/lib/src/raw_types_schemats/ameli.js +2 -0
  54. package/lib/src/raw_types_schemats/debats.d.ts +127 -0
  55. package/lib/src/raw_types_schemats/debats.js +2 -0
  56. package/lib/src/raw_types_schemats/dosleg.d.ts +977 -0
  57. package/lib/src/raw_types_schemats/dosleg.js +2 -0
  58. package/lib/src/raw_types_schemats/questions.d.ts +237 -0
  59. package/lib/src/raw_types_schemats/questions.js +2 -0
  60. package/lib/src/raw_types_schemats/sens.d.ts +2709 -0
  61. package/lib/src/raw_types_schemats/sens.js +2 -0
  62. package/lib/src/rich_types/ameli.d.ts +12 -0
  63. package/lib/src/scripts/data-download.js +13 -13
  64. package/lib/src/scripts/debug_dosleg_query.d.ts +6 -0
  65. package/lib/src/scripts/debug_dosleg_query.js +50 -0
  66. package/lib/src/scripts/retrieve_open_data.js +2 -1
  67. package/lib/src/scripts/shared/data_download.d.ts +43 -0
  68. package/lib/src/scripts/shared/data_download.js +175 -0
  69. package/lib/src/scripts/shared/incremental_import_sql.js +866 -866
  70. package/lib/src/scripts/shared/schema_version.js +85 -84
  71. package/lib/src/scripts/shared/staging_metadata_sql.js +214 -214
  72. package/lib/src/scripts/validate_prefixed_tables.js +12 -12
  73. package/lib/src/server/ameli.js +18 -11
  74. package/lib/src/server/conversion_textes.js +106 -106
  75. package/lib/src/server/datasets.js +1 -0
  76. package/lib/src/server/debats.js +10 -10
  77. package/lib/src/server/documents.js +22 -22
  78. package/lib/src/server/dosleg.js +33 -33
  79. package/lib/src/server/questions.js +10 -10
  80. package/lib/src/server/scrutins.js +3 -3
  81. package/lib/src/server/sens.js +19 -19
  82. package/lib/src/types/agenda.d.ts +45 -0
  83. package/lib/src/types/ameli.d.ts +5 -0
  84. package/lib/src/types/ameli.js +1 -0
  85. package/lib/src/types/compte_rendu.d.ts +83 -0
  86. package/lib/src/types/compte_rendu.js +1 -0
  87. package/lib/src/types/debats.d.ts +2 -0
  88. package/lib/src/types/debats.js +1 -0
  89. package/lib/src/types/dosleg.d.ts +70 -0
  90. package/lib/src/types/dosleg.js +1 -0
  91. package/lib/src/types/questions.d.ts +2 -0
  92. package/lib/src/types/questions.js +1 -0
  93. package/lib/src/types/sens.d.ts +8 -0
  94. package/lib/src/types/sens.js +1 -0
  95. package/lib/src/types/sessions.d.ts +6 -0
  96. package/lib/src/types/sessions.js +19 -0
  97. package/lib/src/types/texte.d.ts +72 -0
  98. package/lib/src/types/texte.js +15 -0
  99. package/lib/src/validators/config.d.ts +9 -0
  100. package/lib/src/validators/config.js +10 -0
  101. package/lib/src/videos/match.js +4 -4
  102. package/lib/tests/dataDownload.test.d.ts +1 -0
  103. package/lib/tests/dataDownload.test.js +217 -0
  104. package/lib/tests/parsers/plf/amendement.test.d.ts +1 -0
  105. package/lib/tests/parsers/plf/amendement.test.js +153 -0
  106. package/lib/tests/parsers/plf/credits.test.js +72 -72
  107. package/lib/tests/parsers/plf/test_utils.js +13 -13
  108. package/lib/tests/parsers/texte.test_utils.js +22 -22
  109. package/package.json +123 -123
  110. package/lib/src/scripts/extract_plf_fixtures.d.ts +0 -1
  111. package/lib/src/scripts/extract_plf_fixtures.js +0 -101
  112. package/lib/tests/plf/credits.test.js +0 -458
  113. package/lib/tests/plf/test_utils.d.ts +0 -20
  114. package/lib/tests/plf/test_utils.js +0 -85
  115. /package/lib/{tests/plf/credits.test.d.ts → src/types/agenda.js} +0 -0
@@ -0,0 +1,83 @@
1
+ export interface CompteRendu {
2
+ uid: string;
3
+ seanceRef: string;
4
+ sessionRef: string;
5
+ metadonnees: Metadonnees;
6
+ contenu: Contenu;
7
+ }
8
+ export interface Metadonnees {
9
+ dateSeance: string;
10
+ dateSeanceJour: string;
11
+ numSeanceJour: string;
12
+ numSeance: string;
13
+ typeAssemblee: "AN" | "SN";
14
+ legislature: string;
15
+ session: string;
16
+ nomFichierJo: string;
17
+ validite: string;
18
+ etat: string;
19
+ diffusion: string;
20
+ version: string;
21
+ environnement: string;
22
+ heureGeneration: Date;
23
+ sommaire?: Sommaire;
24
+ }
25
+ export interface Contenu {
26
+ quantiemes: Quantiemes;
27
+ ouvertureSeance?: Point[] | Point;
28
+ point: Point[] | Point;
29
+ finSeance?: FinSeance;
30
+ paragraphe?: Point[];
31
+ }
32
+ export interface FinSeance {
33
+ point: Point;
34
+ }
35
+ export interface Quantiemes {
36
+ journee: string;
37
+ session: string;
38
+ }
39
+ export interface Point {
40
+ ordre_absolu_seance: string;
41
+ code_grammaire: string;
42
+ roledebat?: string;
43
+ orateurs?: {
44
+ orateur: {
45
+ nom: string;
46
+ id: string;
47
+ qualite: string;
48
+ };
49
+ };
50
+ texte: {
51
+ _: string;
52
+ };
53
+ code_style?: string;
54
+ }
55
+ export interface Texte {
56
+ _?: string;
57
+ id_syceron?: string;
58
+ stime?: string;
59
+ sup?: string;
60
+ lienAdt?: Texte[] | Texte;
61
+ }
62
+ export interface Sommaire {
63
+ presidentSeance: Texte;
64
+ sommaire1: SommaireElement[] | SommaireElement;
65
+ sommaire3?: SommaireElement[] | SommaireElement;
66
+ sommaire2?: SommaireElement[] | SommaireElement;
67
+ para?: Texte[] | Texte;
68
+ }
69
+ export interface SommaireElement {
70
+ valeur_pts_odj: string | undefined;
71
+ titreStruct: TitreStruct;
72
+ para?: Array<Texte | string> | Texte;
73
+ sommaire2?: SommaireElement[] | SommaireElement;
74
+ sommaire3?: SommaireElement[] | SommaireElement;
75
+ presidentSeance?: Texte[] | Texte;
76
+ type_debat?: string;
77
+ }
78
+ export interface TitreStruct {
79
+ id_syceron: string;
80
+ intitule?: string;
81
+ sousIntitule?: string;
82
+ type_debat?: string;
83
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,2 @@
1
+ import { debats as Debat, lecassdeb as LecAssDeb } from "../raw_types_schemats/debats";
2
+ export type { Debat, LecAssDeb };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,70 @@
1
+ import { Ass, Aud as AudRaw, Auteur as AuteurRaw, DateSeance as DateSeanceRaw, Deccoc, Denrap, Docatt as DocattRaw, Ecr as EcrRaw, Etaloi, Lecass as LecassRaw, Lecassrap as LecassrapRaw, Lecture as LectureRaw, Loi as LoiRaw, Org, Oritxt, Qua, Rap as RapRaw, Raporg, Scr, Texte as TexteRaw, Typatt, Typlec, Typloi, Typtxt, Typurl } from "../raw_types/dosleg.js";
2
+ import { TxtAmeli } from "../raw_types/ameli.js";
3
+ import { Debats } from "../raw_types/debats.js";
4
+ import { TxtAmeliCustom } from "./ameli.js";
5
+ export type { Deccoc as DecCoc, Denrap as DenRap, Etaloi as EtaLoi, Oritxt as OriTxt, Raporg as RapOrg, Typatt as TypAtt, Typlec as TypLec, Typloi as TypLoi, Typtxt as TypTxt, Typurl as TypUrl, };
6
+ export interface Aud extends AudRaw {
7
+ org?: Org;
8
+ }
9
+ export interface Auteur extends AuteurRaw {
10
+ qua?: Qua;
11
+ }
12
+ export interface DateSeance extends DateSeanceRaw {
13
+ debat?: Debats;
14
+ scrids?: string[];
15
+ scrs?: Scr[];
16
+ }
17
+ export interface DocAtt extends DocattRaw {
18
+ rap?: RapRaw;
19
+ typatt?: Typatt;
20
+ }
21
+ export interface Ecr extends EcrRaw {
22
+ aut?: Auteur;
23
+ }
24
+ export interface LecAss extends LecassRaw {
25
+ ass?: Ass;
26
+ auds?: Aud[];
27
+ audcles?: AudRaw["audcle"][];
28
+ datesSeances?: DateSeance[];
29
+ datesSeancesCodes?: DateSeanceRaw["code"][];
30
+ debatdatseas: Date[];
31
+ lecassraps?: LecAssRap[];
32
+ lecassrapids?: string[];
33
+ org?: Org;
34
+ texcods: TexteRaw["texcod"][];
35
+ textes?: Texte[];
36
+ }
37
+ export interface LecAssRap extends LecassrapRaw {
38
+ rap?: RapRaw;
39
+ }
40
+ export interface Lecture extends LectureRaw {
41
+ typlec?: Typlec;
42
+ lecassidts?: LecassRaw["lecassidt"][];
43
+ lecasss?: LecAss[];
44
+ }
45
+ export interface Loi extends LoiRaw {
46
+ deccoc?: Deccoc;
47
+ etaloi?: Etaloi;
48
+ lecidts?: LectureRaw["lecidt"][];
49
+ lectures?: Lecture[];
50
+ typloi?: Typloi;
51
+ }
52
+ export interface Rap extends RapRaw {
53
+ denrap?: Denrap;
54
+ docattcles?: DocattRaw["docattcle"][];
55
+ docatts?: DocAtt[];
56
+ ecrnums?: EcrRaw["ecrnum"][];
57
+ ecrs?: Ecr[];
58
+ orgcods?: Raporg["orgcod"][];
59
+ orgs?: Org[];
60
+ }
61
+ export interface Texte extends TexteRaw {
62
+ ecrs?: Ecr[];
63
+ ecrnums?: EcrRaw["ecrnum"][];
64
+ libtypurl?: Typurl["libtypurl"];
65
+ org?: Org;
66
+ oritxt: Oritxt;
67
+ typtxt?: Typtxt;
68
+ txtAmeli?: TxtAmeliCustom;
69
+ txtAmeliId: TxtAmeli["id"];
70
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,2 @@
1
+ import { TamQuestions } from "../raw_types/questions.js";
2
+ export type { TamQuestions as Question };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,8 @@
1
+ export interface Photo {
2
+ chemin: string;
3
+ cheminMosaique: string;
4
+ hauteur: number;
5
+ largeur: number;
6
+ xMosaique: number;
7
+ yMosaique: number;
8
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,6 @@
1
+ export declare const UNDEFINED_SESSION = 0;
2
+ export declare const sessionOptions: readonly [1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
3
+ export declare const sessionOptionsOrAll: readonly [0, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026];
4
+ export type Session = (typeof sessionOptions)[number];
5
+ export type SessionOrAll = (typeof sessionOptionsOrAll)[number];
6
+ export declare function getSessionsFromStart(startSession: SessionOrAll): (1958 | 1959 | 1960 | 1961 | 1962 | 1963 | 1964 | 1965 | 1966 | 1967 | 1968 | 1969 | 1970 | 1971 | 1972 | 1973 | 1974 | 1975 | 1976 | 1977 | 1978 | 1979 | 1980 | 1981 | 1982 | 1983 | 1984 | 1985 | 1986 | 1987 | 1988 | 1989 | 1990 | 1991 | 1992 | 1993 | 1994 | 1995 | 1996 | 1997 | 1998 | 1999 | 2000 | 2001 | 2002 | 2003 | 2004 | 2005 | 2006 | 2007 | 2008 | 2009 | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 | 2018 | 2019 | 2020 | 2021 | 2022 | 2023 | 2024 | 2025 | 2026)[];
@@ -0,0 +1,19 @@
1
+ export const UNDEFINED_SESSION = 0;
2
+ export const sessionOptions = [
3
+ 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976,
4
+ 1977, 1978, 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, 1995,
5
+ 1996, 1997, 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014,
6
+ 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026,
7
+ // TO COMPLETE EVERY YEAR :)
8
+ ];
9
+ export const sessionOptionsOrAll = [UNDEFINED_SESSION, ...sessionOptions];
10
+ export function getSessionsFromStart(startSession) {
11
+ if (startSession === UNDEFINED_SESSION) {
12
+ return Array.from(sessionOptions);
13
+ }
14
+ const sessionIndex = sessionOptions.findIndex((session) => startSession === session);
15
+ if (sessionIndex >= 0) {
16
+ return sessionOptions.slice(sessionIndex);
17
+ }
18
+ return [];
19
+ }
@@ -0,0 +1,72 @@
1
+ import { DocumentResult } from "../loaders.js";
2
+ export declare enum DivisionType {
3
+ tome = 1,
4
+ part = 2,
5
+ book = 3,
6
+ title = 4,
7
+ subtitle = 5,
8
+ chapter = 6,
9
+ section = 7,
10
+ subsection = 8,
11
+ paragraph = 9,
12
+ article = 10,
13
+ alinea = 11,
14
+ division = 12
15
+ }
16
+ export type DivisionTag = keyof typeof DivisionType;
17
+ export interface DocumentMetadata {
18
+ name: string;
19
+ session: number | null | undefined;
20
+ date?: string | null;
21
+ url_expose_des_motifs?: URL;
22
+ url_xml?: URL;
23
+ url_html: URL;
24
+ url_pdf: URL;
25
+ }
26
+ export interface FlatTexte extends Partial<DocumentResult> {
27
+ titre: string | null;
28
+ titre_court: string | null;
29
+ url_dossier_senat: string | null;
30
+ url_dossier_assemblee: string | null;
31
+ date_presentation: Date | null;
32
+ date_depot: Date | null;
33
+ date_publication_xml: Date | null;
34
+ version: Version | null;
35
+ divisions: Division[];
36
+ expose_motifs?: ExposeDesMotifs | null;
37
+ }
38
+ export type Version = "RECT" | "RECT_BIS" | "RECT_TER" | "RECT_QUATER" | "RECT_QUINQUIES";
39
+ export interface Step {
40
+ eId: string;
41
+ date: Date | null;
42
+ type: string | null;
43
+ session: string | null;
44
+ numero: string | null;
45
+ version: Version | null;
46
+ outcome: string | null;
47
+ }
48
+ export interface Division {
49
+ index: number;
50
+ eId: string;
51
+ tag: DivisionTag;
52
+ level: number;
53
+ headings: DivisionContent[];
54
+ }
55
+ export interface Article extends Division {
56
+ alineas: Alinea[];
57
+ }
58
+ export interface DivisionContent {
59
+ text: string | null;
60
+ html?: string | null;
61
+ }
62
+ export interface Alinea extends DivisionContent {
63
+ eId: string;
64
+ heading: DivisionContent;
65
+ pastille: string | null;
66
+ text_avec_liens?: string | null;
67
+ html_avec_liens?: string | null;
68
+ }
69
+ export interface ExposeDesMotifs {
70
+ text: string | null;
71
+ html: string | null;
72
+ }
@@ -0,0 +1,15 @@
1
+ export var DivisionType;
2
+ (function (DivisionType) {
3
+ DivisionType[DivisionType["tome"] = 1] = "tome";
4
+ DivisionType[DivisionType["part"] = 2] = "part";
5
+ DivisionType[DivisionType["book"] = 3] = "book";
6
+ DivisionType[DivisionType["title"] = 4] = "title";
7
+ DivisionType[DivisionType["subtitle"] = 5] = "subtitle";
8
+ DivisionType[DivisionType["chapter"] = 6] = "chapter";
9
+ DivisionType[DivisionType["section"] = 7] = "section";
10
+ DivisionType[DivisionType["subsection"] = 8] = "subsection";
11
+ DivisionType[DivisionType["paragraph"] = 9] = "paragraph";
12
+ DivisionType[DivisionType["article"] = 10] = "article";
13
+ DivisionType[DivisionType["alinea"] = 11] = "alinea";
14
+ DivisionType[DivisionType["division"] = 12] = "division";
15
+ })(DivisionType || (DivisionType = {}));
@@ -0,0 +1,9 @@
1
+ import { z } from "zod";
2
+ export declare const configSchema: z.ZodObject<{
3
+ db: z.ZodObject<{
4
+ host: z.ZodString;
5
+ password: z.ZodString;
6
+ user: z.ZodString;
7
+ port: z.ZodCoercedNumber<unknown>;
8
+ }, z.core.$strip>;
9
+ }, z.core.$strip>;
@@ -0,0 +1,10 @@
1
+ import { z } from "zod";
2
+ const dbSchema = z.object({
3
+ host: z.string().trim().min(1, "Must not be empty"),
4
+ password: z.string().trim().min(1, "Must not be empty"),
5
+ user: z.string().trim().min(1, "Must not be empty"),
6
+ port: z.coerce.number().int().min(0).max(65535),
7
+ });
8
+ export const configSchema = z.object({
9
+ db: dbSchema,
10
+ });
@@ -98,16 +98,16 @@ export async function matchOneReunion(args) {
98
98
  // Accept threshold (best must pass)
99
99
  if (best.score < minAccept) {
100
100
  if (!options["silent"])
101
- console.log(`[threshold] ${agenda.uid} score=${best.score.toFixed(2)}
102
- agenda title="${agenda.titre ?? ""}"
101
+ console.log(`[threshold] ${agenda.uid} score=${best.score.toFixed(2)}
102
+ agenda title="${agenda.titre ?? ""}"
103
103
  best title="${best.vtitle ?? ""}"`);
104
104
  return null;
105
105
  }
106
106
  // Margin (confidence) gate
107
107
  if (second && best.score - second.score < margin) {
108
108
  if (!options["silent"]) {
109
- console.log(`[Margin] ${agenda.uid} score=${best.score.toFixed(2)}
110
- agenda title="${agenda.titre ?? ""}" agenda date="${agenda.startTime ?? ""}"
109
+ console.log(`[Margin] ${agenda.uid} score=${best.score.toFixed(2)}
110
+ agenda title="${agenda.titre ?? ""}" agenda date="${agenda.startTime ?? ""}"
111
111
  best title="${best.vtitle ?? ""}" date="${best.epoch ? new Date(best.epoch * 1000).toISOString() : "null"}"`);
112
112
  console.log("second title=", second.vtitle ?? "", " score=", second.score.toFixed(2), "date=", second.epoch ? new Date(second.epoch * 1000).toISOString() : "null");
113
113
  }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,217 @@
1
+ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import path from "node:path";
4
+ import { afterEach, describe, expect, it } from "vitest";
5
+ import { forwardArgs, parseArgs, readStatusFile, recordFailure, recordSuccess, runDataDownload, STATUS_FILENAME, writeStatusFile, } from "../src/scripts/shared/data_download.js";
6
+ let dataDir;
7
+ afterEach(() => {
8
+ if (dataDir !== undefined) {
9
+ rmSync(dataDir, { force: true, recursive: true });
10
+ }
11
+ });
12
+ function createTempDataDir() {
13
+ dataDir = mkdtempSync(path.join(tmpdir(), "senat-data-download-"));
14
+ return dataDir;
15
+ }
16
+ const fixedDate = new Date("2026-09-08T10:00:00Z");
17
+ function makeExec(respond) {
18
+ const commands = [];
19
+ return {
20
+ commands,
21
+ exec: (command) => {
22
+ commands.push(command);
23
+ return respond(command) ?? { status: 0 };
24
+ },
25
+ };
26
+ }
27
+ describe("parseArgs", () => {
28
+ it("defaults categories to All when none are provided", () => {
29
+ const parsed = parseArgs([]);
30
+ expect(parsed.categories).toEqual(["All"]);
31
+ expect(parsed.dataDir).toBeUndefined();
32
+ expect(parsed.rest).toEqual([]);
33
+ });
34
+ it("extracts the dataDir positional argument and forwards the other options", () => {
35
+ const rawArgs = [
36
+ "--all",
37
+ "/app/senat-data",
38
+ "--fromSession",
39
+ "2022",
40
+ "--fetchDocuments",
41
+ "--parseDocuments",
42
+ "--only-recent",
43
+ "30",
44
+ "--commit",
45
+ "--keepDir",
46
+ ];
47
+ const parsed = parseArgs(rawArgs);
48
+ expect(parsed.dataDir).toBe("/app/senat-data");
49
+ expect(parsed.rest).toEqual([
50
+ "--all",
51
+ "--fromSession",
52
+ "2022",
53
+ "--fetchDocuments",
54
+ "--parseDocuments",
55
+ "--only-recent",
56
+ "30",
57
+ "--commit",
58
+ "--keepDir",
59
+ ]);
60
+ });
61
+ it("collects --categories values including repeated occurrences", () => {
62
+ const parsed = parseArgs(["/data", "--categories", "Ameli", "--categories", "Sens"]);
63
+ expect(parsed.categories).toEqual(["Ameli", "Sens"]);
64
+ expect(parsed.dataDir).toBe("/data");
65
+ expect(parsed.rest).toEqual([]);
66
+ });
67
+ it("supports the -k alias and comma-separated values", () => {
68
+ const parsed = parseArgs(["-k", "Sens,Ameli", "/data"]);
69
+ expect(parsed.categories).toEqual(["Sens", "Ameli"]);
70
+ });
71
+ it("supports the --categories= form", () => {
72
+ const parsed = parseArgs(["--categories=Sens", "/data"]);
73
+ expect(parsed.categories).toEqual(["Sens"]);
74
+ });
75
+ it("does not mistake valued options for the dataDir positional", () => {
76
+ const parsed = parseArgs(["--sudo", "postgres", "--remote", "origin", "true-data", "--verbose"]);
77
+ expect(parsed.dataDir).toBe("true-data");
78
+ expect(parsed.rest).toEqual(["--sudo", "postgres", "--remote", "origin", "--verbose"]);
79
+ });
80
+ });
81
+ describe("forwardArgs", () => {
82
+ it("appends the dataDir then the requested categories to the forwarded options", () => {
83
+ const parsed = parseArgs(["--all", "/app/senat-data", "--categories", "All", "--fromSession", "2022"]);
84
+ const forwarded = forwardArgs(parsed, ["Sens"]);
85
+ expect(forwarded).toBe("--all --fromSession 2022 /app/senat-data --categories Sens");
86
+ });
87
+ });
88
+ describe("dataset status file", () => {
89
+ it("increments consecutiveFailures and keeps firstFailureAt across consecutive failures", () => {
90
+ const statuses = {};
91
+ recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
92
+ recordFailure(statuses, "sens", new Date("2026-09-02T00:00:00Z"), "dump empty again");
93
+ expect(statuses["sens"]).toMatchObject({
94
+ status: "failed",
95
+ consecutiveFailures: 2,
96
+ firstFailureAt: "2026-09-01T00:00:00.000Z",
97
+ lastFailureAt: "2026-09-02T00:00:00.000Z",
98
+ lastError: "dump empty again",
99
+ lastSuccessAt: undefined,
100
+ });
101
+ });
102
+ it("resets the failure streak on success while updating lastSuccessAt", () => {
103
+ const statuses = {};
104
+ recordFailure(statuses, "sens", new Date("2026-09-01T00:00:00Z"), "dump empty");
105
+ recordSuccess(statuses, "sens", new Date("2026-09-02T00:00:00Z"));
106
+ expect(statuses["sens"]).toEqual({
107
+ status: "ok",
108
+ consecutiveFailures: 0,
109
+ lastRunAt: "2026-09-02T00:00:00.000Z",
110
+ lastSuccessAt: "2026-09-02T00:00:00.000Z",
111
+ });
112
+ });
113
+ it("returns an empty state for a missing or corrupted status file", () => {
114
+ const dir = createTempDataDir();
115
+ expect(readStatusFile(path.join(dir, STATUS_FILENAME))).toEqual({});
116
+ const corruptedPath = path.join(dir, "corrupted.json");
117
+ writeFileSync(corruptedPath, "{not json", { encoding: "utf-8" });
118
+ expect(readStatusFile(corruptedPath)).toEqual({});
119
+ });
120
+ it("round-trips statuses through the status file", () => {
121
+ const dir = createTempDataDir();
122
+ const filePath = path.join(dir, STATUS_FILENAME);
123
+ const statuses = {};
124
+ recordFailure(statuses, "sens", fixedDate, "dump empty");
125
+ writeStatusFile(filePath, statuses);
126
+ expect(readStatusFile(filePath)).toEqual(statuses);
127
+ });
128
+ });
129
+ describe("runDataDownload", () => {
130
+ it("continues after a dataset failure, converts only successful categories and exits 0", () => {
131
+ const dir = createTempDataDir();
132
+ const { commands, exec } = makeExec((command) => command.includes("--categories Sens")
133
+ ? { status: 1, error: new Error("Staging schema sens_staging is empty") }
134
+ : undefined);
135
+ const exitCode = runDataDownload({
136
+ rawArgs: [dir, "--fromSession", "2022"],
137
+ exec,
138
+ now: () => fixedDate,
139
+ });
140
+ expect(exitCode).toBe(0);
141
+ const retrieveCommands = commands.filter((command) => command.includes("retrieve_open_data"));
142
+ expect(retrieveCommands).toHaveLength(5);
143
+ for (const command of retrieveCommands) {
144
+ expect(command).toContain("retrieve_open_data.ts --all ");
145
+ }
146
+ expect(retrieveCommands[0]).toContain("--categories Ameli");
147
+ expect(retrieveCommands[4]).toContain("--categories Sens");
148
+ const convertCommand = commands.find((command) => command.includes("convert_data"));
149
+ expect(convertCommand).toContain("--categories Ameli");
150
+ expect(convertCommand).toContain("--categories Debats");
151
+ expect(convertCommand).toContain("--categories DosLeg");
152
+ expect(convertCommand).toContain("--categories Questions");
153
+ expect(convertCommand).not.toContain("--categories Sens");
154
+ const agendaCommand = commands.find((command) => command.includes("retrieve_agenda"));
155
+ expect(agendaCommand).toBe(`cross-env TZ='Etc/UTC' tsx src/scripts/retrieve_agenda.ts ${dir} --fromSession 2022 --parseAgenda --silent`);
156
+ const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
157
+ expect(statuses.sens).toMatchObject({ status: "failed", consecutiveFailures: 1 });
158
+ expect(statuses.ameli).toMatchObject({ status: "ok", consecutiveFailures: 0 });
159
+ });
160
+ it("exits 1 without conversion when all datasets fail", () => {
161
+ const dir = createTempDataDir();
162
+ const { commands, exec } = makeExec(() => ({ status: 1, error: new Error("boom") }));
163
+ const exitCode = runDataDownload({
164
+ rawArgs: [dir],
165
+ exec,
166
+ now: () => fixedDate,
167
+ });
168
+ expect(exitCode).toBe(1);
169
+ expect(commands.filter((command) => command.includes("retrieve_open_data"))).toHaveLength(5);
170
+ expect(commands.find((command) => command.includes("convert_data"))).toBeUndefined();
171
+ expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
172
+ const statuses = JSON.parse(readFileSync(path.join(dir, STATUS_FILENAME), { encoding: "utf-8" }));
173
+ for (const dataset of ["ameli", "debats", "dosleg", "questions", "sens"]) {
174
+ expect(statuses[dataset]).toMatchObject({ status: "failed", consecutiveFailures: 1 });
175
+ }
176
+ });
177
+ it("runs every step when all datasets succeed", () => {
178
+ const dir = createTempDataDir();
179
+ const { commands, exec } = makeExec(() => undefined);
180
+ const exitCode = runDataDownload({
181
+ rawArgs: [dir, "--categories", "All"],
182
+ exec,
183
+ now: () => fixedDate,
184
+ });
185
+ expect(exitCode).toBe(0);
186
+ const convertCommand = commands.find((command) => command.includes("convert_data"));
187
+ for (const category of ["Ameli", "Debats", "DosLeg", "Questions", "Sens"]) {
188
+ expect(convertCommand).toContain(`--categories ${category}`);
189
+ }
190
+ expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeDefined();
191
+ expect(commands.find((command) => command.includes("retrieve_cr_seance"))).toBeDefined();
192
+ expect(commands.find((command) => command.includes("retrieve_cr_commission"))).toBeDefined();
193
+ expect(commands.find((command) => command.includes("retrieve_videos"))).toBeDefined();
194
+ expect(commands.find((command) => command.includes("retrieve_collaborateurs"))).toBeDefined();
195
+ });
196
+ it("stops after convert_data when it fails and propagates its exit status", () => {
197
+ const dir = createTempDataDir();
198
+ const { commands, exec } = makeExec((command) => command.includes("convert_data") ? { status: 3, error: new Error("parse failure") } : undefined);
199
+ const exitCode = runDataDownload({
200
+ rawArgs: [dir],
201
+ exec,
202
+ now: () => fixedDate,
203
+ });
204
+ expect(exitCode).toBe(3);
205
+ expect(commands.find((command) => command.includes("retrieve_agenda"))).toBeUndefined();
206
+ });
207
+ it("exits 1 when no requested category matches a known dataset", () => {
208
+ const { commands, exec } = makeExec(() => undefined);
209
+ const exitCode = runDataDownload({
210
+ rawArgs: ["/data", "--categories", "Unknown"],
211
+ exec,
212
+ now: () => fixedDate,
213
+ });
214
+ expect(exitCode).toBe(1);
215
+ expect(commands).toHaveLength(0);
216
+ });
217
+ });
@@ -0,0 +1 @@
1
+ export {};