@tricoteuses/senat 2.23.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/add-js-extensions-v2.d.ts +1 -0
- package/lib/add-js-extensions-v2.js +23 -0
- package/lib/add-js-extensions.d.ts +1 -0
- package/lib/add-js-extensions.js +17 -0
- package/lib/src/databases_postgres.js +1 -1
- package/lib/src/index.d.ts +24 -13
- package/lib/src/index.js +6 -1
- package/lib/src/loaders.d.ts +12 -12
- package/lib/src/loaders.js +4 -4
- package/lib/src/model/agenda.d.ts +1 -1
- package/lib/src/model/agenda.js +1 -1
- package/lib/src/model/ameli.js +1 -1
- package/lib/src/model/commission.d.ts +2 -2
- package/lib/src/model/commission.js +4 -4
- package/lib/src/model/debats.js +1 -1
- package/lib/src/model/documents.js +1 -1
- package/lib/src/model/dosleg.js +1 -1
- package/lib/src/model/index.d.ts +7 -7
- package/lib/src/model/index.js +7 -7
- package/lib/src/model/questions.js +1 -1
- package/lib/src/model/scrutins.js +1 -1
- package/lib/src/model/seance.d.ts +1 -1
- package/lib/src/model/seance.js +4 -4
- package/lib/src/model/sens.js +1 -1
- package/lib/src/parsers/texte.d.ts +1 -1
- package/lib/src/parsers/texte.js +2 -2
- package/lib/src/raw_types/ameli.d.ts +1651 -803
- package/lib/src/raw_types/ameli.js +1074 -5
- package/lib/src/raw_types/debats.d.ts +353 -180
- package/lib/src/raw_types/debats.js +266 -5
- package/lib/src/raw_types/dosleg.d.ts +2862 -1527
- package/lib/src/raw_types/dosleg.js +2005 -5
- package/lib/src/raw_types/questions.d.ts +671 -395
- package/lib/src/raw_types/questions.js +493 -5
- package/lib/src/raw_types/sens.d.ts +7743 -8148
- package/lib/src/raw_types/sens.js +4691 -5
- package/lib/src/scripts/convert_data.js +10 -10
- package/lib/src/scripts/convert_xml_to_html.js +1 -1
- package/lib/src/scripts/datautil.d.ts +1 -1
- package/lib/src/scripts/retrieve_agenda.js +9 -9
- package/lib/src/scripts/retrieve_cr_commission.js +8 -8
- package/lib/src/scripts/retrieve_cr_seance.d.ts +1 -1
- package/lib/src/scripts/retrieve_cr_seance.js +8 -8
- package/lib/src/scripts/retrieve_documents.d.ts +1 -1
- package/lib/src/scripts/retrieve_documents.js +7 -7
- package/lib/src/scripts/retrieve_open_data.js +155 -49
- package/lib/src/scripts/retrieve_senateurs_photos.js +4 -4
- package/lib/src/scripts/retrieve_videos.js +9 -9
- package/lib/src/scripts/shared/incremental_import_sql.js +1 -1
- package/lib/src/scripts/shared/prefixed_tables.d.ts +6 -3
- package/lib/src/scripts/shared/prefixed_tables.js +22 -16
- package/lib/src/scripts/shared/staging_import.d.ts +1 -1
- package/lib/src/scripts/validate_prefixed_tables.js +8 -9
- package/lib/src/types/ameli.d.ts +4 -4
- package/lib/src/types/debats.d.ts +2 -2
- package/lib/src/types/dosleg.d.ts +39 -39
- package/lib/src/types/questions.d.ts +2 -2
- package/lib/src/types/sens.d.ts +0 -2
- package/lib/src/types/texte.d.ts +1 -1
- package/lib/src/utils/cr_spliting.d.ts +1 -1
- package/lib/src/utils/cr_spliting.js +4 -4
- package/lib/src/utils/nvs-parsing.js +2 -2
- package/lib/src/utils/reunion_odj_building.d.ts +2 -2
- package/lib/src/utils/reunion_odj_building.js +2 -2
- package/lib/src/utils/reunion_parsing.d.ts +2 -2
- package/lib/src/utils/reunion_parsing.js +2 -2
- package/lib/src/utils/scoring.d.ts +3 -3
- package/lib/src/utils/scoring.js +2 -2
- package/lib/src/videos/config.d.ts +1 -1
- package/lib/src/videos/index.d.ts +5 -5
- package/lib/src/videos/index.js +5 -5
- package/lib/src/videos/match.d.ts +2 -2
- package/lib/src/videos/match.js +5 -5
- package/lib/src/videos/pipeline.d.ts +2 -2
- package/lib/src/videos/pipeline.js +4 -4
- package/lib/src/videos/search.d.ts +2 -2
- package/lib/src/videos/search.js +2 -2
- package/lib/tests/incrementalImportSql.test.js +4 -4
- package/lib/tests/prefixedTables.test.js +8 -15
- package/lib/tests/schemaVersion.test.js +1 -1
- package/lib/tests/test_iter_load.test.js +1 -1
- package/lib/tests/validatePrefixedTables.test.js +4 -4
- package/lib/tests/videoMatching.test.js +4 -4
- package/package.json +6 -2
package/lib/src/types/ameli.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
export interface
|
|
4
|
-
subids?:
|
|
1
|
+
import { TxtAmeli } from "../raw_types/ameli.js";
|
|
2
|
+
import { Sub } from "../raw_types/ameli.js";
|
|
3
|
+
export interface TxtAmeliCustom extends TxtAmeli {
|
|
4
|
+
subids?: Sub["id"][];
|
|
5
5
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export type { Debat, LecAssDeb };
|
|
1
|
+
import { Debats, Lecassdeb } from "../raw_types/debats";
|
|
2
|
+
export type { Debats as Debat, Lecassdeb as LecAssDeb };
|
|
@@ -1,70 +1,70 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
export type {
|
|
6
|
-
export interface Aud extends
|
|
1
|
+
import { Ass, Aud as AudRaw, Auteur as AuteurRaw, DateSeance as DateSeanceRaw, Deccoc, Denrap, Docatt as DocattRaw, Ecr as EcrRaw, Etaloi, Lecass as LecassRaw, Lecassrap as LecassrapRaw, Lecture as LectureRaw, Loi as LoiRaw, Org, Oritxt, Qua, Rap as RapRaw, Raporg, Scr, Texte as TexteRaw, Typatt, Typlec, Typloi, Typtxt, Typurl } from "../raw_types/dosleg.js";
|
|
2
|
+
import { TxtAmeli } from "../raw_types/ameli.js";
|
|
3
|
+
import { Debats } from "../raw_types/debats.js";
|
|
4
|
+
import { TxtAmeliCustom } from "./ameli.js";
|
|
5
|
+
export type { Deccoc as DecCoc, Denrap as DenRap, Etaloi as EtaLoi, Oritxt as OriTxt, Raporg as RapOrg, Typatt as TypAtt, Typlec as TypLec, Typloi as TypLoi, Typtxt as TypTxt, Typurl as TypUrl, };
|
|
6
|
+
export interface Aud extends AudRaw {
|
|
7
7
|
org?: Org;
|
|
8
8
|
}
|
|
9
|
-
export interface Auteur extends
|
|
9
|
+
export interface Auteur extends AuteurRaw {
|
|
10
10
|
qua?: Qua;
|
|
11
11
|
}
|
|
12
|
-
export interface DateSeance extends
|
|
13
|
-
debat?:
|
|
12
|
+
export interface DateSeance extends DateSeanceRaw {
|
|
13
|
+
debat?: Debats;
|
|
14
14
|
scrids?: string[];
|
|
15
15
|
scrs?: Scr[];
|
|
16
16
|
}
|
|
17
|
-
export interface DocAtt extends
|
|
18
|
-
rap?:
|
|
19
|
-
typatt?:
|
|
17
|
+
export interface DocAtt extends DocattRaw {
|
|
18
|
+
rap?: RapRaw;
|
|
19
|
+
typatt?: Typatt;
|
|
20
20
|
}
|
|
21
|
-
export interface Ecr extends
|
|
21
|
+
export interface Ecr extends EcrRaw {
|
|
22
22
|
aut?: Auteur;
|
|
23
23
|
}
|
|
24
|
-
export interface LecAss extends
|
|
24
|
+
export interface LecAss extends LecassRaw {
|
|
25
25
|
ass?: Ass;
|
|
26
26
|
auds?: Aud[];
|
|
27
|
-
audcles?:
|
|
27
|
+
audcles?: AudRaw["audcle"][];
|
|
28
28
|
datesSeances?: DateSeance[];
|
|
29
|
-
datesSeancesCodes?:
|
|
29
|
+
datesSeancesCodes?: DateSeanceRaw["code"][];
|
|
30
30
|
debatdatseas: Date[];
|
|
31
31
|
lecassraps?: LecAssRap[];
|
|
32
32
|
lecassrapids?: string[];
|
|
33
33
|
org?: Org;
|
|
34
|
-
texcods:
|
|
34
|
+
texcods: TexteRaw["texcod"][];
|
|
35
35
|
textes?: Texte[];
|
|
36
36
|
}
|
|
37
|
-
export interface LecAssRap extends
|
|
38
|
-
rap?:
|
|
37
|
+
export interface LecAssRap extends LecassrapRaw {
|
|
38
|
+
rap?: RapRaw;
|
|
39
39
|
}
|
|
40
|
-
export interface Lecture extends
|
|
41
|
-
typlec?:
|
|
42
|
-
lecassidts?:
|
|
40
|
+
export interface Lecture extends LectureRaw {
|
|
41
|
+
typlec?: Typlec;
|
|
42
|
+
lecassidts?: LecassRaw["lecassidt"][];
|
|
43
43
|
lecasss?: LecAss[];
|
|
44
44
|
}
|
|
45
|
-
export interface Loi extends
|
|
46
|
-
deccoc?:
|
|
47
|
-
etaloi?:
|
|
48
|
-
lecidts?:
|
|
45
|
+
export interface Loi extends LoiRaw {
|
|
46
|
+
deccoc?: Deccoc;
|
|
47
|
+
etaloi?: Etaloi;
|
|
48
|
+
lecidts?: LectureRaw["lecidt"][];
|
|
49
49
|
lectures?: Lecture[];
|
|
50
|
-
typloi?:
|
|
50
|
+
typloi?: Typloi;
|
|
51
51
|
}
|
|
52
|
-
export interface Rap extends
|
|
53
|
-
denrap?:
|
|
54
|
-
docattcles?:
|
|
52
|
+
export interface Rap extends RapRaw {
|
|
53
|
+
denrap?: Denrap;
|
|
54
|
+
docattcles?: DocattRaw["docattcle"][];
|
|
55
55
|
docatts?: DocAtt[];
|
|
56
|
-
ecrnums?:
|
|
56
|
+
ecrnums?: EcrRaw["ecrnum"][];
|
|
57
57
|
ecrs?: Ecr[];
|
|
58
|
-
orgcods?:
|
|
58
|
+
orgcods?: Raporg["orgcod"][];
|
|
59
59
|
orgs?: Org[];
|
|
60
60
|
}
|
|
61
|
-
export interface Texte extends
|
|
61
|
+
export interface Texte extends TexteRaw {
|
|
62
62
|
ecrs?: Ecr[];
|
|
63
|
-
ecrnums?:
|
|
64
|
-
libtypurl?:
|
|
63
|
+
ecrnums?: EcrRaw["ecrnum"][];
|
|
64
|
+
libtypurl?: Typurl["libtypurl"];
|
|
65
65
|
org?: Org;
|
|
66
|
-
oritxt:
|
|
67
|
-
typtxt?:
|
|
68
|
-
txtAmeli?:
|
|
69
|
-
txtAmeliId:
|
|
66
|
+
oritxt: Oritxt;
|
|
67
|
+
typtxt?: Typtxt;
|
|
68
|
+
txtAmeli?: TxtAmeliCustom;
|
|
69
|
+
txtAmeliId: TxtAmeli["id"];
|
|
70
70
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export type { Question };
|
|
1
|
+
import { TamQuestions } from "../raw_types/questions.js";
|
|
2
|
+
export type { TamQuestions as Question };
|
package/lib/src/types/sens.d.ts
CHANGED
package/lib/src/types/texte.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import path from "path";
|
|
2
2
|
import * as cheerio from "cheerio";
|
|
3
|
-
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
|
|
3
|
+
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders.js";
|
|
4
4
|
import fs from "fs-extra";
|
|
5
|
-
import { sessionStartYearFromDate } from "../model/seance";
|
|
6
|
-
import { frDateToISO, hourShortToStartTime } from "./date";
|
|
7
|
-
import { normalizeSpaces } from "./string_cleaning";
|
|
5
|
+
import { sessionStartYearFromDate } from "../model/seance.js";
|
|
6
|
+
import { frDateToISO, hourShortToStartTime } from "./date.js";
|
|
7
|
+
import { normalizeSpaces } from "./string_cleaning.js";
|
|
8
8
|
function extractWeekStartFromHead($) {
|
|
9
9
|
const og = $('meta[property="og:title"]').attr("content") || $("title").text();
|
|
10
10
|
const m = (og ?? "").toLowerCase().match(/semaine du\s+(\d{1,2}\s+\w+\s+\d{4})/i);
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { XMLParser } from "fast-xml-parser";
|
|
2
|
-
import { dice, normalize } from "./scoring";
|
|
3
|
-
import { decodeHtmlEntities } from "./string_cleaning";
|
|
2
|
+
import { dice, normalize } from "./scoring.js";
|
|
3
|
+
import { decodeHtmlEntities } from "./string_cleaning.js";
|
|
4
4
|
const CHAPTER_MATCH_THRESHOLD = 0.5;
|
|
5
5
|
const xmlParser = new XMLParser({
|
|
6
6
|
ignoreAttributes: false,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import commandLineArgs from "command-line-args";
|
|
2
|
-
import { ActeLegislatif, DossierLegislatifResult } from "../model/dosleg";
|
|
3
|
-
import { AgendaEvent, ReunionOdj } from "../types/agenda";
|
|
2
|
+
import { ActeLegislatif, DossierLegislatifResult } from "../model/dosleg.js";
|
|
3
|
+
import { AgendaEvent, ReunionOdj } from "../types/agenda.js";
|
|
4
4
|
type DossierWithActes = DossierLegislatifResult & {
|
|
5
5
|
actes_legislatifs?: ActeLegislatif[] | null;
|
|
6
6
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { getSessionsFromStart } from "../types/sessions";
|
|
2
|
-
import { iterLoadSenatDossiersLegislatifs } from "../loaders";
|
|
1
|
+
import { getSessionsFromStart } from "../types/sessions.js";
|
|
2
|
+
import { iterLoadSenatDossiersLegislatifs } from "../loaders.js";
|
|
3
3
|
export function buildOdj(events, dossierBySenatUrl) {
|
|
4
4
|
const byObjet = new Map(); // objet -> set de dossier uids
|
|
5
5
|
let codeEtape = null;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DateTime } from "luxon";
|
|
2
2
|
import type { AnyNode } from "domhandler";
|
|
3
|
-
import { AgendaEvent, Reunion } from "../types/agenda";
|
|
4
|
-
import { DossierLegislatifResult } from "../model/dosleg";
|
|
3
|
+
import { AgendaEvent, Reunion } from "../types/agenda.js";
|
|
4
|
+
import { DossierLegislatifResult } from "../model/dosleg.js";
|
|
5
5
|
import * as cheerio from "cheerio";
|
|
6
6
|
type KnownType = "SP" | "COM" | "MC" | "OD" | "ID";
|
|
7
7
|
type DossierBySenatUrl = Record<string, DossierLegislatifResult>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DateTime } from "luxon";
|
|
2
|
-
import { buildOdj } from "./reunion_odj_building";
|
|
3
|
-
import { norm } from "./string_cleaning";
|
|
2
|
+
import { buildOdj } from "./reunion_odj_building.js";
|
|
3
|
+
import { norm } from "./string_cleaning.js";
|
|
4
4
|
const PARIS = "Europe/Paris";
|
|
5
5
|
const STOPWORDS = new Set([
|
|
6
6
|
"de",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { AgendaEvent, Reunion } from "../types/agenda";
|
|
2
|
-
import { VideoScoreSignals, VideoScoreWeights } from "../videos/types";
|
|
3
|
-
import { L1Chapter } from "./nvs-parsing";
|
|
1
|
+
import { AgendaEvent, Reunion } from "../types/agenda.js";
|
|
2
|
+
import { VideoScoreSignals, VideoScoreWeights } from "../videos/types.js";
|
|
3
|
+
import { L1Chapter } from "./nvs-parsing.js";
|
|
4
4
|
export declare function jaccard(a: Set<string>, b: Set<string>): number;
|
|
5
5
|
export declare function jaccardTokenSim(a: string, b: string): number;
|
|
6
6
|
export declare function isNoiseBlock(text: string): boolean;
|
package/lib/src/utils/scoring.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { parseISO } from "./reunion_parsing";
|
|
2
|
-
import { normalizeText } from "./string_cleaning";
|
|
1
|
+
import { parseISO } from "./reunion_parsing.js";
|
|
2
|
+
import { normalizeText } from "./string_cleaning.js";
|
|
3
3
|
export function jaccard(a, b) {
|
|
4
4
|
if (!a.size || !b.size)
|
|
5
5
|
return 0;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export * from "./types";
|
|
2
|
-
export * from "./search";
|
|
3
|
-
export * from "./match";
|
|
4
|
-
export * from "./config";
|
|
5
|
-
export * from "./pipeline";
|
|
1
|
+
export * from "./types.js";
|
|
2
|
+
export * from "./search.js";
|
|
3
|
+
export * from "./match.js";
|
|
4
|
+
export * from "./config.js";
|
|
5
|
+
export * from "./pipeline.js";
|
package/lib/src/videos/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export * from "./types";
|
|
2
|
-
export * from "./search";
|
|
3
|
-
export * from "./match";
|
|
4
|
-
export * from "./config";
|
|
5
|
-
export * from "./pipeline";
|
|
1
|
+
export * from "./types.js";
|
|
2
|
+
export * from "./search.js";
|
|
3
|
+
export * from "./match.js";
|
|
4
|
+
export * from "./config.js";
|
|
5
|
+
export * from "./pipeline.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CommandLineOptions } from "command-line-args";
|
|
2
|
-
import { Reunion } from "../types/agenda";
|
|
3
|
-
import { Candidate, MatchResult, MatchWeights } from "./types";
|
|
2
|
+
import { Reunion } from "../types/agenda.js";
|
|
3
|
+
import { Candidate, MatchResult, MatchWeights } from "./types.js";
|
|
4
4
|
export declare function matchOneReunion(args: {
|
|
5
5
|
agenda: Reunion;
|
|
6
6
|
agendaTs: number | null;
|
package/lib/src/videos/match.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { isAmbiguousTimeOriginal } from "../utils/date";
|
|
2
|
-
import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "../utils/nvs-parsing";
|
|
3
|
-
import { dice, getOrgKey, normalize, scoreVideo } from "../utils/scoring";
|
|
4
|
-
import { SENAT_DATAS_ROOT, weights } from "./config";
|
|
5
|
-
import { fetchBuffer } from "./search";
|
|
1
|
+
import { isAmbiguousTimeOriginal } from "../utils/date.js";
|
|
2
|
+
import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "../utils/nvs-parsing.js";
|
|
3
|
+
import { dice, getOrgKey, normalize, scoreVideo } from "../utils/scoring.js";
|
|
4
|
+
import { SENAT_DATAS_ROOT, weights } from "./config.js";
|
|
5
|
+
import { fetchBuffer } from "./search.js";
|
|
6
6
|
export async function matchOneReunion(args) {
|
|
7
7
|
const { agenda, agendaTs, timeAmbigious, candidates, weights, fetchDataNvs, options } = args;
|
|
8
8
|
if (!options["silent"])
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { Reunion } from "../types/agenda";
|
|
2
|
-
import { BestMatch, LastForVideo, MatchContext } from "./types";
|
|
1
|
+
import { Reunion } from "../types/agenda.js";
|
|
2
|
+
import { BestMatch, LastForVideo, MatchContext } from "./types.js";
|
|
3
3
|
import { CommandLineOptions } from "command-line-args";
|
|
4
4
|
type VideoPipelineOptions = CommandLineOptions & {
|
|
5
5
|
silent?: boolean;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
|
|
2
|
-
import { epochToParisDateTime } from "../utils/date";
|
|
3
|
-
import { SENAT_DATAS_ROOT } from "./config";
|
|
4
|
-
import { fetchText } from "./search";
|
|
1
|
+
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders.js";
|
|
2
|
+
import { epochToParisDateTime } from "../utils/date.js";
|
|
3
|
+
import { SENAT_DATAS_ROOT } from "./config.js";
|
|
4
|
+
import { fetchText } from "./search.js";
|
|
5
5
|
import fs from "fs-extra";
|
|
6
6
|
import fsp from "fs/promises";
|
|
7
7
|
import path from "path";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { CommandLineOptions } from "command-line-args";
|
|
2
|
-
import { Reunion } from "../types/agenda";
|
|
3
|
-
import { Candidate, SearchParams } from "./types";
|
|
2
|
+
import { Reunion } from "../types/agenda.js";
|
|
3
|
+
import { Candidate, SearchParams } from "./types.js";
|
|
4
4
|
export declare function fetchText(url: string): Promise<string | null>;
|
|
5
5
|
export declare function fetchBuffer(url: string): Promise<Buffer | null>;
|
|
6
6
|
export declare function getAgendaType(agenda: Reunion): string;
|
package/lib/src/videos/search.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { toFRDate } from "../utils/date";
|
|
2
|
-
import { MAX_CANDIDATES, SENAT_VIDEOS_SEARCH_AJAX } from "./config";
|
|
1
|
+
import { toFRDate } from "../utils/date.js";
|
|
2
|
+
import { MAX_CANDIDATES, SENAT_VIDEOS_SEARCH_AJAX } from "./config.js";
|
|
3
3
|
import * as cheerio from "cheerio";
|
|
4
4
|
export async function fetchText(url) {
|
|
5
5
|
const res = await fetch(url);
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql, } from "../src/scripts/shared/incremental_import_sql";
|
|
3
|
-
import { stagingSchemaName } from "../src/scripts/shared/prefixed_tables";
|
|
4
|
-
import { buildExportStagingMetadataStatementsQuery } from "../src/scripts/shared/staging_metadata_sql";
|
|
5
|
-
import { isCopyFromStdinLine, rewriteLineForStagingImport } from "../src/scripts/shared/staging_import";
|
|
2
|
+
import { buildIncrementalDatasetImportSql, buildNormalizeStagingSchemaSql, } from "../src/scripts/shared/incremental_import_sql.js";
|
|
3
|
+
import { stagingSchemaName } from "../src/scripts/shared/prefixed_tables.js";
|
|
4
|
+
import { buildExportStagingMetadataStatementsQuery } from "../src/scripts/shared/staging_metadata_sql.js";
|
|
5
|
+
import { isCopyFromStdinLine, rewriteLineForStagingImport } from "../src/scripts/shared/staging_import.js";
|
|
6
6
|
describe("incremental import SQL", () => {
|
|
7
7
|
it("normalizes staging object names before merge", () => {
|
|
8
8
|
const sql = buildNormalizeStagingSchemaSql("dosleg");
|
|
@@ -1,26 +1,19 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import {
|
|
2
|
+
import { buildGeneratedTableManifest, extractPrefixedTableNamesFromGeneratedManifest, prefixedName, senatSchemaName, stagingSchemaName, stripDatasetPrefix, } from "../src/scripts/shared/prefixed_tables.js";
|
|
3
3
|
describe("prefixed table helpers", () => {
|
|
4
4
|
it("builds prefixed table and staging schema names", () => {
|
|
5
5
|
expect(prefixedName("dosleg", "texte")).toBe("dosleg_texte");
|
|
6
6
|
expect(stagingSchemaName("questions")).toBe("questions_staging");
|
|
7
7
|
expect(senatSchemaName).toBe("senat");
|
|
8
8
|
});
|
|
9
|
-
it("
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
const normalized = normalizeGeneratedDefinition(definition, "ameli");
|
|
14
|
-
expect(normalized).toContain("AUTO-GENERATED FILE");
|
|
15
|
-
expect(normalized).not.toContain("AUTO-GENERATED FILE @");
|
|
16
|
-
expect(normalized).toContain("export namespace amdFields");
|
|
17
|
-
expect(normalized).toContain("export interface amd");
|
|
18
|
-
expect(normalized).toContain("export interface ses");
|
|
19
|
-
expect(normalized).not.toContain("ameli_amd");
|
|
9
|
+
it("strips the dataset prefix from generated type names", () => {
|
|
10
|
+
expect(stripDatasetPrefix("ameli_amd", "ameli")).toBe("amd");
|
|
11
|
+
expect(stripDatasetPrefix("ameli_txt_ameli", "ameli")).toBe("txt_ameli");
|
|
12
|
+
expect(stripDatasetPrefix("amd", "ameli")).toBe("amd");
|
|
20
13
|
});
|
|
21
|
-
it("extracts expected prefixed public tables from generated
|
|
22
|
-
const
|
|
23
|
-
expect(
|
|
14
|
+
it("extracts expected prefixed public tables from generated manifests", () => {
|
|
15
|
+
const manifest = buildGeneratedTableManifest("ameli", ["ameli_amd", "ameli_ses", "ameli_txt_ameli"]);
|
|
16
|
+
expect(extractPrefixedTableNamesFromGeneratedManifest(manifest, "ameli")).toEqual([
|
|
24
17
|
"ameli_amd",
|
|
25
18
|
"ameli_ses",
|
|
26
19
|
"ameli_txt_ameli",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "../src/scripts/shared/schema_version";
|
|
2
|
+
import { buildEnsureSchemaVersionTableSql, buildIncrementSchemaVersionSql, buildSchemaStructureFingerprintQuery, } from "../src/scripts/shared/schema_version.js";
|
|
3
3
|
describe("schema version SQL", () => {
|
|
4
4
|
it("creates the senat.version table and seeds version zero", () => {
|
|
5
5
|
const sql = buildEnsureSchemaVersionTableSql("senat");
|
|
@@ -2,7 +2,7 @@ import fs from "fs";
|
|
|
2
2
|
import os from "os";
|
|
3
3
|
import path from "path";
|
|
4
4
|
import { describe, it, expect } from "vitest";
|
|
5
|
-
import { COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatComptesRendusSeances } from "../src/loaders";
|
|
5
|
+
import { COMPTES_RENDUS_FOLDER, DATA_TRANSFORMED_FOLDER, iterLoadSenatComptesRendusSeances } from "../src/loaders.js";
|
|
6
6
|
describe("iterLoadSenatComptesRendusSeances", () => {
|
|
7
7
|
it("loads comptes rendus from transformed/session folder", () => {
|
|
8
8
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "senat-cr-"));
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { readFileSync } from "fs";
|
|
2
2
|
import { describe, expect, it } from "vitest";
|
|
3
|
-
import { datasets } from "../src/datasets";
|
|
4
|
-
import {
|
|
3
|
+
import { datasets } from "../src/datasets.js";
|
|
4
|
+
import { extractPrefixedTableNamesFromGeneratedManifest, getGeneratedTableManifestPath, } from "../src/scripts/shared/prefixed_tables.js";
|
|
5
5
|
describe("generated raw types coverage", () => {
|
|
6
6
|
it("maps each generated dataset definition to prefixed senat tables", () => {
|
|
7
7
|
for (const dataset of Object.values(datasets)) {
|
|
8
|
-
const
|
|
9
|
-
const tableNames =
|
|
8
|
+
const manifest = readFileSync(getGeneratedTableManifestPath(dataset.database), { encoding: "utf8" });
|
|
9
|
+
const tableNames = extractPrefixedTableNamesFromGeneratedManifest(manifest, dataset.database);
|
|
10
10
|
expect(tableNames.length).toBeGreaterThan(0);
|
|
11
11
|
expect(tableNames.every((tableName) => tableName.startsWith(`${dataset.database}_`))).toBe(true);
|
|
12
12
|
}
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
import { describe, it, expect } from "vitest";
|
|
8
8
|
import * as fs from "node:fs/promises";
|
|
9
9
|
import * as path from "node:path";
|
|
10
|
-
import { isAmbiguousTimeOriginal, toFRDate } from "../src/utils/date";
|
|
11
|
-
import { dice, getOrgKey, normalize, scoreVideo } from "../src/utils/scoring";
|
|
12
|
-
import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "../src/utils/nvs-parsing";
|
|
13
|
-
import { extractCandidatesFromSearchHtml, fetchAllSearchPages, fetchBuffer, getAgendaType, SENAT_DATAS_ROOT, } from "../src/videos";
|
|
10
|
+
import { isAmbiguousTimeOriginal, toFRDate } from "../src/utils/date.js";
|
|
11
|
+
import { dice, getOrgKey, normalize, scoreVideo } from "../src/utils/scoring.js";
|
|
12
|
+
import { buildSenatVodMasterM3u8FromNvs, getLevel1Chapters, parseDataNvs } from "../src/utils/nvs-parsing.js";
|
|
13
|
+
import { extractCandidatesFromSearchHtml, fetchAllSearchPages, fetchBuffer, getAgendaType, SENAT_DATAS_ROOT, } from "../src/videos/index.js";
|
|
14
14
|
const LIVE_CACHE_DIR = path.join(process.cwd(), "tests", ".cache", "video-matching-live");
|
|
15
15
|
const FIXTURES_ROOT = path.join(process.cwd(), "tests", "fixtures", "data");
|
|
16
16
|
const GOLD_PATH = path.join(FIXTURES_ROOT, "expected-video-matching.json");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tricoteuses/senat",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.1",
|
|
4
4
|
"description": "Handle French Sénat's open data",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"France",
|
|
@@ -46,6 +46,7 @@
|
|
|
46
46
|
"access": "public"
|
|
47
47
|
},
|
|
48
48
|
"scripts": {
|
|
49
|
+
"postinstall": "patch-package",
|
|
49
50
|
"build": "tsc",
|
|
50
51
|
"build:types": "tsc --emitDeclarationOnly",
|
|
51
52
|
"data:convert_data": "tsx src/scripts/convert_data.ts",
|
|
@@ -86,7 +87,6 @@
|
|
|
86
87
|
"zod": "^4.3.5"
|
|
87
88
|
},
|
|
88
89
|
"devDependencies": {
|
|
89
|
-
"@typed-code/schemats": "^5.0.1",
|
|
90
90
|
"@types/cheerio": "^1.0.0",
|
|
91
91
|
"@types/command-line-args": "^5.0.0",
|
|
92
92
|
"@types/fs-extra": "^11.0.4",
|
|
@@ -99,6 +99,10 @@
|
|
|
99
99
|
"cross-env": "^10.1.0",
|
|
100
100
|
"eslint": "^9.39.2",
|
|
101
101
|
"globals": "^17.0.0",
|
|
102
|
+
"kanel": "^4.0.1",
|
|
103
|
+
"kanel-zod": "^4.0.0",
|
|
104
|
+
"patch-package": "^8.0.1",
|
|
105
|
+
"postinstall-postinstall": "^2.1.0",
|
|
102
106
|
"prettier": "^3.5.3",
|
|
103
107
|
"tsx": "^4.21.0",
|
|
104
108
|
"typescript": "^5.9.3",
|